From 16d39d3d07bb01cd051c4b0cff958720ef22f8e9 Mon Sep 17 00:00:00 2001
From: ddasarathan <ddasarathan@confluent.io>
Date: Mon, 11 Apr 2022 15:55:46 -0700
Subject: [PATCH 001/356] Add Jenkinsfile to the setup

---
 Jenkinsfile | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 Jenkinsfile

diff --git a/Jenkinsfile b/Jenkinsfile
new file mode 100644
index 000000000..780ab4ccd
--- /dev/null
+++ b/Jenkinsfile
@@ -0,0 +1,4 @@
+common {
+     cron = null
+     timeoutHours = 2
+ }
\ No newline at end of file

From 7803ae45c50bda8291cbfac32ad1bba8d2bdac6f Mon Sep 17 00:00:00 2001
From: Charles Provencher <cprovencher@confluent.io>
Date: Mon, 18 Mar 2024 05:14:45 -0400
Subject: [PATCH 002/356] Migration: Jenkins -> Semaphore (#1)

* First step migration Jenkins->Semaphore

* Enable cc-service-bot git plugin

* Add lang_version: unknown

* chore: update repo semaphore project

* chore: update repo semaphore config

* Delete Jenkinsfile

* chore: update repo semaphore config

* Update semaphore.yml to open all java.base

* updated the sem-version java to 8

---------

Co-authored-by: KGaneshDatta <124683189+KGaneshDatta@users.noreply.github.com>
---
 .semaphore/project.yml   | 43 +++++++++++++++++++++
 .semaphore/semaphore.yml | 80 ++++++++++++++++++++++++++++++++++++++++
 Jenkinsfile              |  4 --
 service.yml              | 10 +++++
 4 files changed, 133 insertions(+), 4 deletions(-)
 create mode 100644 .semaphore/project.yml
 create mode 100644 .semaphore/semaphore.yml
 delete mode 100644 Jenkinsfile
 create mode 100644 service.yml

diff --git a/.semaphore/project.yml b/.semaphore/project.yml
new file mode 100644
index 000000000..38cd017d8
--- /dev/null
+++ b/.semaphore/project.yml
@@ -0,0 +1,43 @@
+# This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common
+# template and configurations in service.yml.
+# Modifications in this file will be overwritten by generated content in the nightly run.
+# For more information, please refer to the page:
+# https://confluentinc.atlassian.net/wiki/spaces/Foundations/pages/2871296194/Add+SemaphoreCI
+apiVersion: v1alpha
+kind: Project
+metadata:
+  name: snowflake-ingest-java
+  description: ""
+spec:
+  visibility: private
+  repository:
+    url: git@github.com:confluentinc/snowflake-ingest-java.git
+    run_on:
+    - branches
+    - pull_requests
+    pipeline_file: .semaphore/semaphore.yml
+    integration_type: github_app
+    status:
+      pipeline_files:
+      - path: .semaphore/semaphore.yml
+        level: pipeline
+    whitelist:
+      branches:
+      - master
+      - main
+      - /^\d+\.\d+\.x$/
+      - /^gh-readonly-queue.*/
+  custom_permissions: true
+  debug_permissions:
+  - empty
+  - default_branch
+  - non_default_branch
+  - pull_request
+  - forked_pull_request
+  - tag
+  attach_permissions:
+  - default_branch
+  - non_default_branch
+  - pull_request
+  - forked_pull_request
+  - tag
diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
new file mode 100644
index 000000000..6518b2ee5
--- /dev/null
+++ b/.semaphore/semaphore.yml
@@ -0,0 +1,80 @@
+# This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common
+# template and configurations in service.yml.
+# Any modifications made to ths file will be overwritten by the generated content in nightly runs.
+# For more information, please refer to the page:
+# https://confluentinc.atlassian.net/wiki/spaces/Foundations/pages/2871296194/Add+SemaphoreCI
+version: v1.0
+name: build-test-release
+agent:
+  machine:
+    type: s1-prod-ubuntu20-04-amd64-1
+
+fail_fast:
+  cancel:
+    when: "true"
+
+execution_time_limit:
+  hours: 1
+
+queue:
+  - when: "branch != 'master' and branch !~ '[0-9]+\\.[0-9]+\\.x'"
+    processing: parallel
+
+global_job_config:
+  prologue:
+    commands:
+      - checkout
+      - . sem-version java 8
+      - . cache-maven restore
+
+blocks:
+  - name: Test
+    dependencies: []
+    run:
+      # don't run the tests on non-functional changes...
+      when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/']})"
+    task:
+      jobs:
+        - name: Test
+          commands:
+            - echo 'export JAVA_OPTS="$JAVA_OPTS --add-opens java.base/java.lang=ALL-UNNAMED"' >> ~/.bashrc
+            - source ~/.bashrc
+            - mvn -U -Dmaven.wagon.http.retryHandler.count=10 --batch-mode --no-transfer-progress clean verify install dependency:analyze validate
+            - . cache-maven store
+      epilogue:
+        always:
+          commands:
+            - . publish-test-results
+            - artifact push workflow target/test-results
+
+  - name: Release
+    dependencies: ["Test"]
+    run:
+      when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.x'"
+    task:
+      jobs:
+        - name: Release
+          commands:
+            - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
+            - git fetch --unshallow || true
+            - mvn -U -Dmaven.wagon.http.retryHandler.count=10 --batch-mode -DaltDeploymentRepository=confluent-codeartifact-internal::default::https://confluent-519856050701.d.codeartifact.us-west-2.amazonaws.com/maven/maven-snapshots/
+              -DrepositoryId=confluent-codeartifact-internal deploy -DskipTests
+
+
+after_pipeline:
+  task:
+    agent:
+      machine:
+        type: s1-prod-ubuntu20-04-arm64-0
+    jobs:
+      - name: Metrics
+        commands:
+          - emit-ci-metrics -p -a test-results
+      - name: Publish Test Results
+        commands:
+          - test-results gen-pipeline-report
+      - name: SonarQube
+        commands:
+          - checkout
+          - sem-version java 11
+          - emit-sonarqube-data -a test-results
diff --git a/Jenkinsfile b/Jenkinsfile
deleted file mode 100644
index 780ab4ccd..000000000
--- a/Jenkinsfile
+++ /dev/null
@@ -1,4 +0,0 @@
-common {
-     cron = null
-     timeoutHours = 2
- }
\ No newline at end of file
diff --git a/service.yml b/service.yml
new file mode 100644
index 000000000..50d192d6e
--- /dev/null
+++ b/service.yml
@@ -0,0 +1,10 @@
+name: snowflake-ingest-java
+lang: unknown
+lang_version: unknown
+codeowners:
+  enable: true
+semaphore:
+  enable: true
+  pipeline_type: cp
+git:
+  enable: true

From 29232f9a0baf16ecead258affd2ca614ea03e5b7 Mon Sep 17 00:00:00 2001
From: Confluent Jenkins Bot <jenkins@confluent.io>
Date: Sun, 26 Jan 2025 00:29:23 -0500
Subject: [PATCH 003/356] chore: update repo by service bot (#2)

* removing generated internal project.yml

* chore: update repo semaphore config

---------

Co-authored-by: service-bot-app[bot] <189278048+service-bot-app[bot]@users.noreply.github.com>
---
 .semaphore/project.yml   | 43 ----------------------------------------
 .semaphore/semaphore.yml | 12 +++++------
 2 files changed, 5 insertions(+), 50 deletions(-)
 delete mode 100644 .semaphore/project.yml

diff --git a/.semaphore/project.yml b/.semaphore/project.yml
deleted file mode 100644
index 38cd017d8..000000000
--- a/.semaphore/project.yml
+++ /dev/null
@@ -1,43 +0,0 @@
-# This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common
-# template and configurations in service.yml.
-# Modifications in this file will be overwritten by generated content in the nightly run.
-# For more information, please refer to the page:
-# https://confluentinc.atlassian.net/wiki/spaces/Foundations/pages/2871296194/Add+SemaphoreCI
-apiVersion: v1alpha
-kind: Project
-metadata:
-  name: snowflake-ingest-java
-  description: ""
-spec:
-  visibility: private
-  repository:
-    url: git@github.com:confluentinc/snowflake-ingest-java.git
-    run_on:
-    - branches
-    - pull_requests
-    pipeline_file: .semaphore/semaphore.yml
-    integration_type: github_app
-    status:
-      pipeline_files:
-      - path: .semaphore/semaphore.yml
-        level: pipeline
-    whitelist:
-      branches:
-      - master
-      - main
-      - /^\d+\.\d+\.x$/
-      - /^gh-readonly-queue.*/
-  custom_permissions: true
-  debug_permissions:
-  - empty
-  - default_branch
-  - non_default_branch
-  - pull_request
-  - forked_pull_request
-  - tag
-  attach_permissions:
-  - default_branch
-  - non_default_branch
-  - pull_request
-  - forked_pull_request
-  - tag
diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
index 6518b2ee5..6d943e3ed 100644
--- a/.semaphore/semaphore.yml
+++ b/.semaphore/semaphore.yml
@@ -24,7 +24,7 @@ global_job_config:
   prologue:
     commands:
       - checkout
-      - . sem-version java 8
+      - . set-cp-java-version
       - . cache-maven restore
 
 blocks:
@@ -37,8 +37,7 @@ blocks:
       jobs:
         - name: Test
           commands:
-            - echo 'export JAVA_OPTS="$JAVA_OPTS --add-opens java.base/java.lang=ALL-UNNAMED"' >> ~/.bashrc
-            - source ~/.bashrc
+            - . sem-pint -c
             - mvn -U -Dmaven.wagon.http.retryHandler.count=10 --batch-mode --no-transfer-progress clean verify install dependency:analyze validate
             - . cache-maven store
       epilogue:
@@ -46,6 +45,7 @@ blocks:
           commands:
             - . publish-test-results
             - artifact push workflow target/test-results
+            - artifact push workflow target
 
   - name: Release
     dependencies: ["Test"]
@@ -55,12 +55,9 @@ blocks:
       jobs:
         - name: Release
           commands:
-            - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
-            - git fetch --unshallow || true
             - mvn -U -Dmaven.wagon.http.retryHandler.count=10 --batch-mode -DaltDeploymentRepository=confluent-codeartifact-internal::default::https://confluent-519856050701.d.codeartifact.us-west-2.amazonaws.com/maven/maven-snapshots/
               -DrepositoryId=confluent-codeartifact-internal deploy -DskipTests
 
-
 after_pipeline:
   task:
     agent:
@@ -77,4 +74,5 @@ after_pipeline:
         commands:
           - checkout
           - sem-version java 11
-          - emit-sonarqube-data -a test-results
+          - artifact pull workflow target
+          - emit-sonarqube-data --run_only_sonar_scan

From ff47b803fb176b8aaf45e43f76aead9e013120b4 Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Fri, 7 Feb 2025 10:24:14 +0530
Subject: [PATCH 004/356] add ci files (#23)

---
 .semaphore/project.yml | 44 ++++++++++++++++++++++++++++++++++++++++++
 service.yml            | 11 +++++++++--
 2 files changed, 53 insertions(+), 2 deletions(-)
 create mode 100644 .semaphore/project.yml

diff --git a/.semaphore/project.yml b/.semaphore/project.yml
new file mode 100644
index 000000000..3b5c02330
--- /dev/null
+++ b/.semaphore/project.yml
@@ -0,0 +1,44 @@
+# This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common
+# template and configurations in service.yml.
+# Modifications in this file will be overwritten by generated content in the nightly run.
+# For more information, please refer to the page:
+# https://confluentinc.atlassian.net/wiki/spaces/Foundations/pages/2871296194/Add+SemaphoreCI
+apiVersion: v1alpha
+kind: Project
+metadata:
+  name: snowflake-ingest-java
+  description: ""
+spec:
+  visibility: private
+  repository:
+    url: git@github.com:confluentinc/snowflake-ingest-java.git
+    run_on:
+      - branches
+      - pull_requests
+    pipeline_file: .semaphore/semaphore.yml
+    integration_type: github_app
+    status:
+      pipeline_files:
+        - path: .semaphore/semaphore.yml
+          level: pipeline
+    whitelist:
+      branches:
+        - master
+        - main
+        - /^\d+\.\d+\.x$/
+        - /v\d+\.\d+\.\d+\-hotfix\-x/
+        - /^gh-readonly-queue.*/
+  custom_permissions: true
+  debug_permissions:
+    - empty
+    - default_branch
+    - non_default_branch
+    - pull_request
+    - forked_pull_request
+    - tag
+  attach_permissions:
+    - default_branch
+    - non_default_branch
+    - pull_request
+    - forked_pull_request
+    - tag
\ No newline at end of file
diff --git a/service.yml b/service.yml
index 50d192d6e..34f43e43b 100644
--- a/service.yml
+++ b/service.yml
@@ -1,10 +1,17 @@
 name: snowflake-ingest-java
-lang: unknown
-lang_version: unknown
+lang: java
+lang_version: 8
 codeowners:
   enable: true
 semaphore:
   enable: true
   pipeline_type: cp
+  cve_scan: true
+  branches:
+    - master
+    - main
+    - /^\d+\.\d+\.x$/
+    - /v\d+\.\d+\.\d+\-hotfix\-x/
+    - /^gh-readonly-queue.*/
 git:
   enable: true

From 43d9e7b1c012b6b74af5152b6ea736d033959b24 Mon Sep 17 00:00:00 2001
From: "service-bot-app[bot]"
 <189278048+service-bot-app[bot]@users.noreply.github.com>
Date: Fri, 7 Feb 2025 06:57:05 +0000
Subject: [PATCH 005/356] chore: update repo by service bot (#3)

* removing generated internal project.yml

* chore: update repo semaphore config

* chore: update sonar-project.properties to reconfigure sonarqube scanning.

---------

Co-authored-by: service-bot-app[bot] <189278048+service-bot-app[bot]@users.noreply.github.com>
---
 .semaphore/project.yml   | 44 ----------------------------------------
 .semaphore/semaphore.yml |  3 ++-
 sonar-project.properties |  9 ++++++++
 3 files changed, 11 insertions(+), 45 deletions(-)
 delete mode 100644 .semaphore/project.yml
 create mode 100644 sonar-project.properties

diff --git a/.semaphore/project.yml b/.semaphore/project.yml
deleted file mode 100644
index 3b5c02330..000000000
--- a/.semaphore/project.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-# This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common
-# template and configurations in service.yml.
-# Modifications in this file will be overwritten by generated content in the nightly run.
-# For more information, please refer to the page:
-# https://confluentinc.atlassian.net/wiki/spaces/Foundations/pages/2871296194/Add+SemaphoreCI
-apiVersion: v1alpha
-kind: Project
-metadata:
-  name: snowflake-ingest-java
-  description: ""
-spec:
-  visibility: private
-  repository:
-    url: git@github.com:confluentinc/snowflake-ingest-java.git
-    run_on:
-      - branches
-      - pull_requests
-    pipeline_file: .semaphore/semaphore.yml
-    integration_type: github_app
-    status:
-      pipeline_files:
-        - path: .semaphore/semaphore.yml
-          level: pipeline
-    whitelist:
-      branches:
-        - master
-        - main
-        - /^\d+\.\d+\.x$/
-        - /v\d+\.\d+\.\d+\-hotfix\-x/
-        - /^gh-readonly-queue.*/
-  custom_permissions: true
-  debug_permissions:
-    - empty
-    - default_branch
-    - non_default_branch
-    - pull_request
-    - forked_pull_request
-    - tag
-  attach_permissions:
-    - default_branch
-    - non_default_branch
-    - pull_request
-    - forked_pull_request
-    - tag
\ No newline at end of file
diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
index 6d943e3ed..2b4847230 100644
--- a/.semaphore/semaphore.yml
+++ b/.semaphore/semaphore.yml
@@ -24,7 +24,7 @@ global_job_config:
   prologue:
     commands:
       - checkout
-      - . set-cp-java-version
+      - sem-version java 8
       - . cache-maven restore
 
 blocks:
@@ -39,6 +39,7 @@ blocks:
           commands:
             - . sem-pint -c
             - mvn -U -Dmaven.wagon.http.retryHandler.count=10 --batch-mode --no-transfer-progress clean verify install dependency:analyze validate
+            - cve-scan
             - . cache-maven store
       epilogue:
         always:
diff --git a/sonar-project.properties b/sonar-project.properties
new file mode 100644
index 000000000..c7eb73003
--- /dev/null
+++ b/sonar-project.properties
@@ -0,0 +1,9 @@
+### service-bot sonarqube plugin managed file
+sonar.coverage.exclusions=**/test/**/*,**/tests/**/*,**/mock/**/*,**/mocks/**/*,**/*mock*,**/*test*
+sonar.coverage.jacoco.xmlReportPaths=**/jacoco.xml
+sonar.cpd.exclusions=**/test/**/*,**/tests/**/*,**/mock/**/*,**/mocks/**/*,**/*mock*,**/*test*
+sonar.exclusions=**/*.pb.*,**/mk-include/**/*
+sonar.java.binaries=.
+sonar.language=java
+sonar.projectKey=snowflake-ingest-java
+sonar.sources=.

From 956a46dcd52733c9105410051dfc63d3aa0b8a92 Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Fri, 7 Feb 2025 14:37:20 +0530
Subject: [PATCH 006/356] Add code artifact section in service.yml (#29)

---
 service.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/service.yml b/service.yml
index 34f43e43b..dfb3dd22c 100644
--- a/service.yml
+++ b/service.yml
@@ -15,3 +15,7 @@ semaphore:
     - /^gh-readonly-queue.*/
 git:
   enable: true
+code_artifact:
+  enable: true
+  package_paths:
+    - maven-snapshots/maven/io.confluent/snowflake-ingest-sdk
\ No newline at end of file

From 826147b2268944bb6a66a2b9229f6271cdc249b2 Mon Sep 17 00:00:00 2001
From: "service-bot-app[bot]"
 <189278048+service-bot-app[bot]@users.noreply.github.com>
Date: Fri, 14 Feb 2025 07:10:41 +0000
Subject: [PATCH 007/356] chore: update repo semaphore config (#27)

Co-authored-by: service-bot-app[bot] <foo-bar+service-bot-app[bot]@users.noreply.github.com>
---
 .semaphore/semaphore.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
index 2b4847230..79e4adaa6 100644
--- a/.semaphore/semaphore.yml
+++ b/.semaphore/semaphore.yml
@@ -7,7 +7,7 @@ version: v1.0
 name: build-test-release
 agent:
   machine:
-    type: s1-prod-ubuntu20-04-amd64-1
+    type: s1-prod-ubuntu24-04-amd64-1
 
 fail_fast:
   cancel:
@@ -63,7 +63,7 @@ after_pipeline:
   task:
     agent:
       machine:
-        type: s1-prod-ubuntu20-04-arm64-0
+        type: s1-prod-ubuntu24-04-arm64-0
     jobs:
       - name: Metrics
         commands:

From a5c05372e4b0cdef17eae29264fd3d6cd3caeeef Mon Sep 17 00:00:00 2001
From: "service-bot-app[bot]"
 <189278048+service-bot-app[bot]@users.noreply.github.com>
Date: Sat, 22 Feb 2025 07:14:43 +0000
Subject: [PATCH 008/356] chore: update repo semaphore config (#31)

Co-authored-by: service-bot-app[bot] <foo-bar+service-bot-app[bot]@users.noreply.github.com>
Co-authored-by: service-bot-app[bot] <189278048+service-bot-app[bot]@users.noreply.github.com>

From f7c3dea7049c0783c07eea70ca677e1fa15e6aa4 Mon Sep 17 00:00:00 2001
From: "service-bot-app[bot]"
 <189278048+service-bot-app[bot]@users.noreply.github.com>
Date: Sat, 26 Apr 2025 08:32:02 +0000
Subject: [PATCH 009/356] chore: update repo semaphore config (#33)

Co-authored-by: service-bot-app[bot] <foo-bar+service-bot-app[bot]@users.noreply.github.com>
---
 .semaphore/semaphore.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
index 79e4adaa6..ee1d13ad0 100644
--- a/.semaphore/semaphore.yml
+++ b/.semaphore/semaphore.yml
@@ -32,7 +32,7 @@ blocks:
     dependencies: []
     run:
       # don't run the tests on non-functional changes...
-      when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/']})"
+      when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/'], default_branch: 'master'})"
     task:
       jobs:
         - name: Test

From b6b62afa1c92646393c35024f7b7ae3fe9bd17ee Mon Sep 17 00:00:00 2001
From: "service-bot-app[bot]"
 <189278048+service-bot-app[bot]@users.noreply.github.com>
Date: Sun, 4 May 2025 08:15:00 +0000
Subject: [PATCH 010/356] chore: update repo semaphore config (#34)

Co-authored-by: service-bot-app[bot] <foo-bar+service-bot-app[bot]@users.noreply.github.com>
Co-authored-by: service-bot-app[bot] <189278048+service-bot-app[bot]@users.noreply.github.com>

From 9d2d15cf9b7bf2bb2ee5e2e839b2266197b5bcee Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Tue, 5 Apr 2022 16:56:14 -0700
Subject: [PATCH 011/356] Release version 1.0.2-beta (#150)

Release note:

1. Fix a blocker issue that would cause data querying failure if a client has multiple channels that belongs to different tables
2. Add JMX metrics
3. Remove usage of instanceof during type checking
4. Fix an issue that file registration would fail for columns with all NULLs
5. Support SDK with proxy server
6. Support Date input as a String
---
 pom.xml                                                     | 6 +++---
 .../net/snowflake/ingest/connection/RequestBuilder.java     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pom.xml b/pom.xml
index 9a747959f..04a5d74e5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
     <!-- Arifact name and version information -->
     <groupId>net.snowflake</groupId>
     <artifactId>snowflake-ingest-sdk</artifactId>
-    <version>1.0.1-beta</version>
+    <version>1.0.2-beta</version>
     <packaging>jar</packaging>
     <name>Snowflake Ingest SDK</name>
     <description>Snowflake Ingest SDK</description>
@@ -17,7 +17,7 @@
     <licenses>
       <license>
         <name>The Apache Software License, Version 2.0</name>
-        <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+        <url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
       </license>
     </licenses>
 
@@ -32,7 +32,7 @@
 
     <scm>
       <connection>scm:git:git://github.com/snowflakedb/snowflake-ingest-java</connection>
-      <url>http://github.com/snowflakedb/snowflake-ingest-java/tree/master</url>
+      <url>https://github.com/snowflakedb/snowflake-ingest-java/tree/master</url>
     </scm>
 
     <!-- Set our Language Level to Java 8 -->
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 44d846b94..034cebeb6 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -127,7 +127,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.0.1-beta";
+  public static final String DEFAULT_VERSION = "1.0.2-beta";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From a099d70ffa8d3c9d0ec8cf31c45c73773095c11a Mon Sep 17 00:00:00 2001
From: Jay Patel <jay.patel@snowflake.com>
Date: Tue, 5 Apr 2022 17:20:11 -0700
Subject: [PATCH 012/356] Enable ignored tests which run against sfctest0 prod1
 account (#108)

- New snowpipe client APIs are enabled for corresponding account which runs E2E tests
---
 src/test/java/net/snowflake/ingest/SimpleIngestIT.java | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
index 02c32aca2..cdcb0e52b 100644
--- a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
@@ -33,7 +33,6 @@
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 
 /** Example ingest sdk integration test */
@@ -329,7 +328,6 @@ private void verifyDefaultUserAgent(
     }
   }
 
-  @Ignore
   @Test
   public void testConfigureClientHappyCase() throws Exception {
     final String userAgentSuffix = "kafka-provider/NONE";
@@ -338,7 +336,6 @@ public void testConfigureClientHappyCase() throws Exception {
     assertEquals(0L, configureClientResponse.getClientSequencer().longValue());
   }
 
-  @Ignore
   @Test
   public void testConfigureClientNoPipeFound() throws Exception {
     final String userAgentSuffix = "kafka-provider/NONE";
@@ -353,7 +350,6 @@ public void testConfigureClientNoPipeFound() throws Exception {
     }
   }
 
-  @Ignore
   @Test
   public void testGetClientStatusHappyCase() throws Exception {
     final String userAgentSuffix = "kafka-provider/NONE";
@@ -364,7 +360,6 @@ public void testGetClientStatusHappyCase() throws Exception {
     assertNull(clientStatusResponse.getOffsetToken());
   }
 
-  @Ignore
   @Test
   public void testGetClientStatusNoPipeFound() throws Exception {
     final String userAgentSuffix = "kafka-provider/NONE";
@@ -379,7 +374,6 @@ public void testGetClientStatusNoPipeFound() throws Exception {
     }
   }
 
-  @Ignore
   @Test
   public void testIngestFilesWithClientInfo() throws Exception {
 
@@ -415,7 +409,6 @@ public void testIngestFilesWithClientInfo() throws Exception {
     assertEquals(offsetToken, clientStatusResponse.getOffsetToken());
   }
 
-  @Ignore
   @Test
   public void testIngestFilesWithClientInfoWithOldClientSequencer() throws Exception {
 
@@ -477,7 +470,6 @@ public void testIngestFilesWithClientInfoWithOldClientSequencer() throws Excepti
     }
   }
 
-  @Ignore
   @Test
   public void testIngestFilesWithClientInfoWithNoClientSequencer() throws Exception {
     // first lets call configure client API

From 8d5292d4511b65bffab4b3cdc64d19af703407f4 Mon Sep 17 00:00:00 2001
From: Jay Patel <jay.patel@snowflake.com>
Date: Wed, 6 Apr 2022 20:23:57 -0700
Subject: [PATCH 013/356] no-snow fix error message printing (#157)

* no-snow fix error message printing

- prints the message on KC if parameter for message code 0024 is not
  passed
  "retry for getLatestCommittedOffsetToken. Retry no:1, message:Get channel status request failed: {0}."

* Add to other APIs too
---
 .../internal/SnowflakeStreamingIngestClientInternal.java    | 6 +++---
 .../ingest/streaming/internal/StreamingIngestStage.java     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 8fa3b109b..51f74d116 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -305,7 +305,7 @@ public SnowflakeStreamingIngestChannelInternal openChannel(OpenChannelRequest re
 
       return channel;
     } catch (IOException | IngestResponseException e) {
-      throw new SFException(e, ErrorCode.OPEN_CHANNEL_FAILURE);
+      throw new SFException(e, ErrorCode.OPEN_CHANNEL_FAILURE, e.getMessage());
     }
   }
 
@@ -342,7 +342,7 @@ ChannelsStatusResponse getChannelsStatus(List<SnowflakeStreamingIngestChannelInt
 
       return response;
     } catch (IOException | IngestResponseException e) {
-      throw new SFException(e, ErrorCode.CHANNEL_STATUS_FAILURE);
+      throw new SFException(e, ErrorCode.CHANNEL_STATUS_FAILURE, e.getMessage());
     }
   }
 
@@ -383,7 +383,7 @@ void registerBlobs(List<BlobMetadata> blobs) {
         throw new SFException(ErrorCode.REGISTER_BLOB_FAILURE, response.getMessage());
       }
     } catch (IOException | IngestResponseException e) {
-      throw new SFException(e, ErrorCode.REGISTER_BLOB_FAILURE);
+      throw new SFException(e, ErrorCode.REGISTER_BLOB_FAILURE, e.getMessage());
     }
 
     logger.logDebug(
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index 94a871b6f..448e773ac 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -315,7 +315,7 @@ private Map<String, Object> makeClientConfigureCall(Map<Object, Object> payload)
       }
       return response;
     } catch (IngestResponseException e) {
-      throw new SFException(e, ErrorCode.CLIENT_CONFIGURE_FAILURE);
+      throw new SFException(e, ErrorCode.CLIENT_CONFIGURE_FAILURE, e.getMessage());
     }
   }
 

From d965ac6369f854cf95adf3488028428a5879abdb Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Fri, 8 Apr 2022 13:56:41 +0000
Subject: [PATCH 014/356] @snow SNOW-572229 Streaning Ingest: Manual padding of
 compressed chunk for encryption

---
 .../streaming/internal/BlobBuilder.java       | 13 +++++++---
 .../streaming/internal/FlushService.java      | 25 +++++++++++--------
 .../net/snowflake/ingest/utils/Constants.java |  4 +--
 3 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
index 9cb366a2c..da1f3dfea 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
@@ -25,6 +25,7 @@
 import java.util.zip.GZIPOutputStream;
 import javax.xml.bind.DatatypeConverter;
 import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
 
 /**
  * Build a single blob file that contains file header plus data. The header will be a
@@ -52,10 +53,13 @@ class BlobBuilder {
    *
    * @param filePath blob file full path
    * @param chunkData uncompressed chunk data
-   * @return compressed chunk data
+   * @param blockSizeToAlignTo block size to align to for encryption
+   * @return padded compressed chunk data, aligned to blockSizeToAlignTo, and actual length of
+   *     compressed data before padding at the end
    * @throws IOException
    */
-  static byte[] compress(String filePath, ByteArrayOutputStream chunkData) throws IOException {
+  static Pair<byte[], Integer> compress(
+      String filePath, ByteArrayOutputStream chunkData, int blockSizeToAlignTo) throws IOException {
     int uncompressedSize = chunkData.size();
     ByteArrayOutputStream compressedOutputStream = new ByteArrayOutputStream(uncompressedSize);
     try (GZIPOutputStream gzipOutputStream = new GZIPOutputStream(compressedOutputStream, true)) {
@@ -85,7 +89,10 @@ static byte[] compress(String filePath, ByteArrayOutputStream chunkData) throws
         firstCompressedSize,
         doubleCompressedSize);
 
-    return compressedOutputStream.toByteArray();
+    int compressedSize = compressedOutputStream.size();
+    int paddingSize = blockSizeToAlignTo - compressedSize % blockSizeToAlignTo;
+    compressedOutputStream.write(new byte[paddingSize]);
+    return new Pair<>(compressedOutputStream.toByteArray(), compressedSize);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 60c72463d..7a9f43e94 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -476,8 +476,18 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
       }
 
       if (!channelsMetadataList.isEmpty()) {
-        // Compress the chunk data
-        byte[] compressedChunkData = BlobBuilder.compress(filePath, chunkData);
+        // Compress the chunk data and pad it for encryption.
+        // Encryption needs padding to the ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES
+        // to align with decryption on the Snowflake query path starting from this chunk offset.
+        // The padding does not have arrow data and not compressed.
+        // Hence, the actual chunk size is smaller by the padding size.
+        // The compression on the Snowflake query path needs the correct size of the compressed
+        // data.
+        Pair<byte[], Integer> compressionResult =
+            BlobBuilder.compress(
+                filePath, chunkData, Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES);
+        byte[] compressedAndPaddedChunkData = compressionResult.getFirst();
+        int compressedChunkLength = compressionResult.getSecond();
 
         // Encrypt the compressed chunk data, the encryption key is derived using the key from
         // server with the full blob path.
@@ -486,15 +496,8 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
         // TODO: address alignment for the header SNOW-557866
         long iv = curDataSize / Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES;
         byte[] encryptedCompressedChunkData =
-            Cryptor.encrypt(compressedChunkData, firstChannel.getEncryptionKey(), filePath, iv);
-
-        // Encryption adds padding to the ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES
-        // to align with decryption on the Snowflake query path starting from this chunk offset.
-        // The padding does not have arrow data and not compressed.
-        // Hence, the actual chunk size is smaller by the padding size.
-        // The compression on the Snowflake query path needs the correct size of the compressed
-        // data.
-        int compressedChunkLength = compressedChunkData.length;
+            Cryptor.encrypt(
+                compressedAndPaddedChunkData, firstChannel.getEncryptionKey(), filePath, iv);
 
         // Compute the md5 of the chunk data
         String md5 = BlobBuilder.computeMD5(encryptedCompressedChunkData, compressedChunkLength);
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index c91be1a4a..41307d6e8 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -44,8 +44,8 @@ public class Constants {
   public static final int COMMIT_RETRY_INTERVAL_IN_MS = 500;
   public static final int RESPONSE_ROW_SEQUENCER_IS_COMMITTED =
       26; // Don't change, should match server side
-  public static final String ENCRYPTION_ALGORITHM = "AES/CTR/PKCS7Padding";
-  public static final long ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES = 16;
+  public static final String ENCRYPTION_ALGORITHM = "AES/CTR/NoPadding";
+  public static final int ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES = 16;
 
   // Channel level constants
   public static final String CHANNEL_STATUS_ENDPOINT = "/v1/streaming/channels/status/";

From b0368bd4fbceb89982e7a3fe9dcd983bc53461cc Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Fri, 8 Apr 2022 14:59:26 -0700
Subject: [PATCH 015/356] add retry on 429 (#160)

NO-SNOW: add retry logic on 429 (TOO_MANY_REQUESTS)
---
 src/main/java/net/snowflake/ingest/utils/HttpUtil.java | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index 9b70c8bf5..8e36e192e 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -130,6 +130,8 @@ private static ServiceUnavailableRetryStrategy getServiceUnavailableRetryStrateg
     return new ServiceUnavailableRetryStrategy() {
       private int executionCount = 0;
       final int REQUEST_TIMEOUT = 408;
+      final int TOO_MANY_REQUESTS = 429;
+      final int SERVER_ERRORS = 500;
 
       @Override
       public boolean retryRequest(
@@ -141,7 +143,9 @@ public boolean retryRequest(
           return false;
         }
         boolean needNextRetry =
-            (statusCode == REQUEST_TIMEOUT || statusCode >= 500)
+            (statusCode == REQUEST_TIMEOUT
+                    || statusCode == TOO_MANY_REQUESTS
+                    || statusCode >= SERVER_ERRORS)
                 && executionCount < MAX_RETRIES + 1;
         if (needNextRetry) {
           long interval = (1 << executionCount) * 1000;

From a69b15644cf2224507787b9f1b18aadc4d683c2a Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Fri, 8 Apr 2022 18:34:30 -0700
Subject: [PATCH 016/356] NO-SNOW: fix regression after the JMX metrics change
 (#161)

Fixing a NullPointerException when the JMX metrics collection is enabled
---
 ...nowflakeStreamingIngestClientInternal.java |  3 +-
 .../streaming/internal/StreamingIngestIT.java | 52 ++++++++++++++++++-
 2 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 51f74d116..090676a05 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -585,7 +585,7 @@ List<SnowflakeStreamingIngestChannelInternal> verifyChannelsAreFullyCommitted(
   }
 
   /**
-   * Get get ParameterProvider with configurable parameters
+   * Get ParameterProvider with configurable parameters
    *
    * @return ParameterProvider used by the client
    */
@@ -628,6 +628,7 @@ private void registerMetricsForClient() {
     jmxReporter.start();
 
     // add JVM and thread metrics too
+    jvmMemoryAndThreadMetrics = new MetricRegistry();
     jvmMemoryAndThreadMetrics.register(
         MetricRegistry.name("jvm", "memory"), new MemoryUsageGaugeSet());
     jvmMemoryAndThreadMetrics.register(
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 547dfc4b2..009127827 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -26,6 +26,7 @@
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
 import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.ParameterProvider;
 import net.snowflake.ingest.utils.SFException;
 import org.junit.After;
 import org.junit.Assert;
@@ -137,7 +138,7 @@ public void testSimpleIngest() throws Exception {
           if (BLOB_NO_HEADER && COMPRESS_BLOB_TWICE) {
             Assert.assertEquals(3445, client.blobSizeHistogram.getSnapshot().getMax());
           } else if (BLOB_NO_HEADER) {
-            Assert.assertEquals(3579, client.blobSizeHistogram.getSnapshot().getMax());
+            Assert.assertEquals(3600, client.blobSizeHistogram.getSnapshot().getMax());
           } else if (COMPRESS_BLOB_TWICE) {
             Assert.assertEquals(3981, client.blobSizeHistogram.getSnapshot().getMax());
           } else {
@@ -151,6 +152,55 @@ public void testSimpleIngest() throws Exception {
     Assert.fail("Row sequencer not updated before timeout");
   }
 
+  @Test
+  public void testParameterOverrides() throws Exception {
+    Map<String, Object> parameterMap = new HashMap<>();
+    parameterMap.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS_MAP_KEY, 30L);
+    parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_MAP_KEY, 50L);
+    parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_MAP_KEY, 1L);
+    parameterMap.put(ParameterProvider.INSERT_THROTTLE_INTERVAL_IN_MILLIS_MAP_KEY, 1L);
+    parameterMap.put(ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS_MAP_KEY, true);
+    client =
+        (SnowflakeStreamingIngestClientInternal)
+            SnowflakeStreamingIngestClientFactory.builder("testParameterOverridesClient")
+                .setProperties(prop)
+                .setParameterOverrides(parameterMap)
+                .build();
+
+    OpenChannelRequest request1 =
+        OpenChannelRequest.builder("CHANNEL")
+            .setDBName(TEST_DB)
+            .setSchemaName(TEST_SCHEMA)
+            .setTableName(TEST_TABLE)
+            .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+            .build();
+
+    // Open a streaming ingest channel from the given client
+    SnowflakeStreamingIngestChannel channel1 = client.openChannel(request1);
+    for (int val = 0; val < 10; val++) {
+      Map<String, Object> row = new HashMap<>();
+      row.put("c1", Integer.toString(val));
+      verifyInsertValidationResponse(channel1.insertRow(row, Integer.toString(val)));
+    }
+
+    for (int i = 1; i < 15; i++) {
+      if (channel1.getLatestCommittedOffsetToken() != null
+          && channel1.getLatestCommittedOffsetToken().equals("9")) {
+        ResultSet result =
+            jdbcConnection
+                .createStatement()
+                .executeQuery(
+                    String.format(
+                        "select count(*) from %s.%s.%s", TEST_DB, TEST_SCHEMA, TEST_TABLE));
+        result.next();
+        Assert.assertEquals(10, result.getLong(1));
+        return;
+      }
+      Thread.sleep(500);
+    }
+    Assert.fail("Row sequencer not updated before timeout");
+  }
+
   @Test
   public void testInterleavedIngest() {
     Consumer<IntConsumer> iter =

From d0b42d05e78da4bd4dff6aa1f7df23411086eabb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Bregu=C5=82a?= <kamil.bregula@snowflake.com>
Date: Thu, 14 Apr 2022 23:29:09 +0200
Subject: [PATCH 017/356] NO-SNOW: Format code (#155)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Hi.

I'm lazy and don't want to install a code formatting tool, and then bother manually running it, so I created a script that installs and formats code with one simple ./format.sh command.

Best regards,
Kamil Breguła
---
 .github/workflows/GoogleJavaFormat.yml | 13 +++++++------
 .gitignore                             |  1 +
 README.md                              | 10 +++++-----
 format.sh                              | 21 +++++++++++++++++++++
 4 files changed, 34 insertions(+), 11 deletions(-)
 create mode 100755 format.sh

diff --git a/.github/workflows/GoogleJavaFormat.yml b/.github/workflows/GoogleJavaFormat.yml
index f932ec319..4e321b914 100644
--- a/.github/workflows/GoogleJavaFormat.yml
+++ b/.github/workflows/GoogleJavaFormat.yml
@@ -11,10 +11,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: GoogleJavaFormat
-        id: googlejavaformat
-        uses: axel-op/googlejavaformat-action@v3
+      - uses: actions/setup-java@v2
         with:
-          skipCommit: true
-          version: 1.10.0
-          args: "-n --set-exit-if-changed"
+          distribution: 'adopt'
+          java-version: '11'
+      - name: Format code
+        run: ./format.sh
+      - if: ${{ failure() }}
+        run: git diff --color=always
diff --git a/.gitignore b/.gitignore
index ce014b839..7cda7192a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,4 @@ profile.properties
 src/main/resources/log4j.properties
 src/test/resources/log4j.properties
 testOutput/
+.cache/
diff --git a/README.md b/README.md
index 51aecbc53..17e635442 100644
--- a/README.md
+++ b/README.md
@@ -95,9 +95,9 @@ you would need to remove the following scope limits in pom.xml
     -   Here is the link for documentation [Key Pair
             Generator](https://docs.snowflake.net/manuals/user-guide/python-connector-example.html#using-key-pair-authentication)
 
-# Google Java Format
+# Code style
 
-- Download the formatter jar file from https://github.com/google/google-java-format, then run it with 
-```
-java -jar ~/path-to/google-java-format-1.10.0-all-deps.jar  -i $(find . -type f -name "*.java" | grep ".*/src/.*java")
-```
+We use [Google Java format](https://github.com/google/google-java-format) to format the code. To format all files, run:
+```bash
+./format.sh
+````
diff --git a/format.sh b/format.sh
new file mode 100755
index 000000000..0cade08f5
--- /dev/null
+++ b/format.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$SCRIPT_DIR"
+
+DOWNLOAD_URL="https://github.com/google/google-java-format/releases/download/v1.10.0/google-java-format-1.10.0-all-deps.jar"
+JAR_FILE="./.cache/google-java-format-1.10.0-all-deps.jar"
+
+if [ ! -f "${JAR_FILE}" ]; then
+  mkdir -p "$(dirname "${JAR_FILE}")"
+  echo "Downloading Google Java format to ${JAR_FILE}"
+  curl -# -L --fail "${DOWNLOAD_URL}" --output "${JAR_FILE}"
+fi
+
+if ! command -v java > /dev/null; then
+  echo "Java not installed."
+  exit 1
+fi
+echo "Running Google Java Format"
+find ./src -type f -name "*.java" -print0 | xargs -0 java -jar "${JAR_FILE}" --replace --set-exit-if-changed && echo "OK"

From 205915a25c72d2941f317d65006039fc24dc65a9 Mon Sep 17 00:00:00 2001
From: Matt Naides <63469689+sfc-gh-mnaides@users.noreply.github.com>
Date: Tue, 19 Apr 2022 14:58:42 -0700
Subject: [PATCH 018/356] SNOW-556750 retry certain streaming ingest specific
 retries (#158)

* add ingest specific code retry

* add unit tests

* checkpoint

* checkpoint

* update test

* use Pair instead of bespoke pojo

* pr feedback

* test update
---
 .../internal/ChannelsStatusResponse.java      |   3 +-
 .../internal/OpenChannelResponse.java         |   3 +-
 .../internal/RegisterBlobResponse.java        |   2 +-
 ...nowflakeStreamingIngestClientInternal.java | 176 ++++++--
 .../internal/StreamingIngestResponse.java     |   9 +
 .../internal/StreamingIngestStage.java        |  31 +-
 .../internal/StreamingIngestUtils.java        | 118 +++++
 .../net/snowflake/ingest/utils/Constants.java |   5 +
 .../SnowflakeStreamingIngestClientTest.java   | 413 ++++++++++++++++++
 .../internal/StreamingIngestStageTest.java    |   9 +-
 .../internal/StreamingIngestUtilsTest.java    | 165 +++++++
 11 files changed, 884 insertions(+), 50 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestResponse.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelsStatusResponse.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelsStatusResponse.java
index 0f49d3a5f..96a24581d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelsStatusResponse.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelsStatusResponse.java
@@ -8,7 +8,7 @@
 import java.util.List;
 
 /** Class used to serialize a response for the channels status endpoint */
-class ChannelsStatusResponse {
+class ChannelsStatusResponse extends StreamingIngestResponse {
 
   static class ChannelStatusResponseDTO {
 
@@ -75,6 +75,7 @@ void setStatusCode(Long statusCode) {
   }
 
   @JsonProperty("status_code")
+  @Override
   Long getStatusCode() {
     return this.statusCode;
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/OpenChannelResponse.java b/src/main/java/net/snowflake/ingest/streaming/internal/OpenChannelResponse.java
index c103314ca..beb0eb0c4 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/OpenChannelResponse.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/OpenChannelResponse.java
@@ -8,7 +8,7 @@
 import java.util.List;
 
 /** Response to the OpenChannelRequest */
-class OpenChannelResponse {
+class OpenChannelResponse extends StreamingIngestResponse {
   private Long statusCode;
   private String message;
   private String dbName;
@@ -27,6 +27,7 @@ void setStatusCode(Long statusCode) {
     this.statusCode = statusCode;
   }
 
+  @Override
   Long getStatusCode() {
     return this.statusCode;
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterBlobResponse.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterBlobResponse.java
index 6c4baad66..bcfae3355 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterBlobResponse.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterBlobResponse.java
@@ -8,7 +8,7 @@
 import java.util.List;
 
 /** Response to the RegisterBlobRequest */
-class RegisterBlobResponse {
+class RegisterBlobResponse extends StreamingIngestResponse {
   private Long statusCode;
   private String message;
   private List<BlobRegisterStatus> blobsStatus;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 090676a05..7d7a5d374 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -7,12 +7,17 @@
 import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CHANNEL_STATUS;
 import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_OPEN_CHANNEL;
 import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_REGISTER_BLOB;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.sleepForRetry;
 import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.COMMIT_MAX_RETRY_COUNT;
 import static net.snowflake.ingest.utils.Constants.COMMIT_RETRY_INTERVAL_IN_MS;
 import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
+import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
 import static net.snowflake.ingest.utils.Constants.OPEN_CHANNEL_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_ROW_SEQUENCER_IS_COMMITTED;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
 import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JMX_METRIC_PREFIX;
@@ -38,9 +43,11 @@
 import java.security.spec.InvalidKeySpecException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -50,13 +57,13 @@
 import javax.management.ObjectName;
 import net.snowflake.ingest.connection.IngestResponseException;
 import net.snowflake.ingest.connection.RequestBuilder;
-import net.snowflake.ingest.connection.ServiceResponseHandler;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.HttpUtil;
 import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
 import net.snowflake.ingest.utils.ParameterProvider;
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.SnowflakeURL;
@@ -260,12 +267,14 @@ public SnowflakeStreamingIngestChannelInternal openChannel(OpenChannelRequest re
       payload.put("role", this.role);
 
       OpenChannelResponse response =
-          ServiceResponseHandler.unmarshallStreamingIngestResponse(
-              httpClient.execute(
-                  requestBuilder.generateStreamingIngestPostRequest(
-                      payload, OPEN_CHANNEL_ENDPOINT, "open channel")),
+          executeWithRetries(
               OpenChannelResponse.class,
-              STREAMING_OPEN_CHANNEL);
+              OPEN_CHANNEL_ENDPOINT,
+              payload,
+              "open channel",
+              STREAMING_OPEN_CHANNEL,
+              httpClient,
+              requestBuilder);
 
       // Check for Snowflake specific response code
       if (response.getStatusCode() != RESPONSE_SUCCESS) {
@@ -327,13 +336,16 @@ ChannelsStatusResponse getChannelsStatus(List<SnowflakeStreamingIngestChannelInt
       request.setRequestId(this.flushService.getClientPrefix() + "_" + counter.getAndIncrement());
 
       String payload = objectMapper.writeValueAsString(request);
+
       ChannelsStatusResponse response =
-          ServiceResponseHandler.unmarshallStreamingIngestResponse(
-              httpClient.execute(
-                  requestBuilder.generateStreamingIngestPostRequest(
-                      payload, CHANNEL_STATUS_ENDPOINT, "channel status")),
+          executeWithRetries(
               ChannelsStatusResponse.class,
-              STREAMING_CHANNEL_STATUS);
+              CHANNEL_STATUS_ENDPOINT,
+              payload,
+              "channel status",
+              STREAMING_CHANNEL_STATUS,
+              httpClient,
+              requestBuilder);
 
       // Check for Snowflake specific response code
       if (response.getStatusCode() != RESPONSE_SUCCESS) {
@@ -352,10 +364,21 @@ ChannelsStatusResponse getChannelsStatus(List<SnowflakeStreamingIngestChannelInt
    * @param blobs list of uploaded blobs
    */
   void registerBlobs(List<BlobMetadata> blobs) {
+    this.registerBlobs(blobs, 0);
+  }
+
+  /**
+   * Register the uploaded blobs to a Snowflake table
+   *
+   * @param blobs list of uploaded blobs
+   * @param executionCount Number of times this call has been attempted, used to track retries
+   */
+  void registerBlobs(List<BlobMetadata> blobs, final int executionCount) {
     logger.logDebug(
-        "Register blob request start for blob={}, client={}",
+        "Register blob request start for blob={}, client={}, executionCount={}",
         blobs.stream().map(BlobMetadata::getPath).collect(Collectors.toList()),
-        this.name);
+        this.name,
+        executionCount);
 
     RegisterBlobResponse response = null;
     try {
@@ -366,20 +389,23 @@ void registerBlobs(List<BlobMetadata> blobs) {
       payload.put("role", this.role);
 
       response =
-          ServiceResponseHandler.unmarshallStreamingIngestResponse(
-              httpClient.execute(
-                  requestBuilder.generateStreamingIngestPostRequest(
-                      payload, REGISTER_BLOB_ENDPOINT, "register blob")),
+          executeWithRetries(
               RegisterBlobResponse.class,
-              STREAMING_REGISTER_BLOB);
+              REGISTER_BLOB_ENDPOINT,
+              payload,
+              "register blob",
+              STREAMING_REGISTER_BLOB,
+              httpClient,
+              requestBuilder);
 
       // Check for Snowflake specific response code
       if (response.getStatusCode() != RESPONSE_SUCCESS) {
         logger.logDebug(
-            "Register blob request failed for blob={}, client={}, message={}",
+            "Register blob request failed for blob={}, client={}, message={}, executionCount={}",
             blobs.stream().map(BlobMetadata::getPath).collect(Collectors.toList()),
             this.name,
-            response.getMessage());
+            response.getMessage(),
+            executionCount);
         throw new SFException(ErrorCode.REGISTER_BLOB_FAILURE, response.getMessage());
       }
     } catch (IOException | IngestResponseException e) {
@@ -387,11 +413,13 @@ void registerBlobs(List<BlobMetadata> blobs) {
     }
 
     logger.logDebug(
-        "Register blob request succeeded for blob={}, client={}",
+        "Register blob request succeeded for blob={}, client={}, executionCount={}",
         blobs.stream().map(BlobMetadata::getPath).collect(Collectors.toList()),
-        this.name);
+        this.name,
+        executionCount);
 
-    // Invalidate any channels that returns a failure status code
+    // We will retry any blob chunks that were rejected becuase internal Snowflake queues are full
+    Set<ChunkRegisterStatus> queueFullChunks = new HashSet<>();
     response
         .getBlobsStatus()
         .forEach(
@@ -405,21 +433,97 @@ void registerBlobs(List<BlobMetadata> blobs) {
                                 .forEach(
                                     channelStatus -> {
                                       if (channelStatus.getStatusCode() != RESPONSE_SUCCESS) {
-                                        logger.logWarn(
-                                            "Channel has been invalidated because of failure"
-                                                + " response, name={}, channel sequencer={},"
-                                                + " status code={}",
-                                            channelStatus.getChannelName(),
-                                            channelStatus.getChannelSequencer(),
-                                            channelStatus.getStatusCode());
-                                        channelCache.invalidateChannelIfSequencersMatch(
-                                            chunkStatus.getDBName(),
-                                            chunkStatus.getSchemaName(),
-                                            chunkStatus.getTableName(),
-                                            channelStatus.getChannelName(),
-                                            channelStatus.getChannelSequencer());
+                                        // If the chunk queue is full, we wait and retry the chunks
+                                        if ((channelStatus.getStatusCode()
+                                                    == RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL
+                                                || channelStatus.getStatusCode()
+                                                    == RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST)
+                                            && executionCount
+                                                < MAX_STREAMING_INGEST_API_CHANNEL_RETRY) {
+                                          queueFullChunks.add(chunkStatus);
+                                        } else {
+                                          logger.logWarn(
+                                              "Channel has been invalidated because of failure"
+                                                  + " response, name={}, channel sequencer={},"
+                                                  + " status code={}, executionCount={}",
+                                              channelStatus.getChannelName(),
+                                              channelStatus.getChannelSequencer(),
+                                              channelStatus.getStatusCode(),
+                                              executionCount);
+                                          channelCache.invalidateChannelIfSequencersMatch(
+                                              chunkStatus.getDBName(),
+                                              chunkStatus.getSchemaName(),
+                                              chunkStatus.getTableName(),
+                                              channelStatus.getChannelName(),
+                                              channelStatus.getChannelSequencer());
+                                        }
                                       }
                                     })));
+
+    if (!queueFullChunks.isEmpty()) {
+      logger.logDebug(
+          "Retrying registerBlobs request, blobs={}, retried_chunks={}, executionCount={}",
+          blobs,
+          queueFullChunks,
+          executionCount);
+      List<BlobMetadata> retryBlobs = this.getRetryBlobs(queueFullChunks, blobs);
+      if (retryBlobs.isEmpty()) {
+        throw new SFException(ErrorCode.INTERNAL_ERROR, "Failed to retry queue full chunks");
+      }
+      sleepForRetry(executionCount);
+      this.registerBlobs(retryBlobs, executionCount + 1);
+    }
+  }
+
+  /**
+   * Constructs a new register blobs payload consisting of chunks that were rejected by a prior
+   * registration attempt
+   *
+   * @param queueFullChunks ChunkRegisterStatus values for the chunks that had been rejected
+   * @param blobs List<BlobMetadata> from the prior registration call
+   * @return a new List<BlobMetadata> for only chunks matching queueFullChunks
+   */
+  List<BlobMetadata> getRetryBlobs(
+      Set<ChunkRegisterStatus> queueFullChunks, List<BlobMetadata> blobs) {
+    /*
+    If a channel returns a RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL statusCode then all channels in the same chunk
+    will have that statusCode.  Here we collect all channels with RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL and use
+    them to pull out the chunks to retry from blobs
+     */
+    Set<Pair<String, Long>> queueFullKeys =
+        queueFullChunks.stream()
+            .flatMap(
+                chunkRegisterStatus -> {
+                  return chunkRegisterStatus.getChannelsStatus().stream()
+                      .map(
+                          channelStatus ->
+                              new Pair<String, Long>(
+                                  channelStatus.getChannelName(),
+                                  channelStatus.getChannelSequencer()));
+                })
+            .collect(Collectors.toSet());
+    List<BlobMetadata> retryBlobs = new ArrayList<>();
+    blobs.forEach(
+        blobMetadata -> {
+          List<ChunkMetadata> relevantChunks =
+              blobMetadata.getChunks().stream()
+                  .filter(
+                      chunkMetadata ->
+                          chunkMetadata.getChannels().stream()
+                              .map(
+                                  channelMetadata ->
+                                      new Pair(
+                                          channelMetadata.getChannelName(),
+                                          channelMetadata.getClientSequencer()))
+                              .anyMatch(channelKey -> queueFullKeys.contains(channelKey)))
+                  .collect(Collectors.toList());
+          if (!relevantChunks.isEmpty()) {
+            retryBlobs.add(
+                new BlobMetadata(blobMetadata.getPath(), blobMetadata.getMD5(), relevantChunks));
+          }
+        });
+
+    return retryBlobs;
   }
 
   /** Close the client, which will flush first and then release all the resources */
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestResponse.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestResponse.java
new file mode 100644
index 000000000..1ec01fceb
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestResponse.java
@@ -0,0 +1,9 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+abstract class StreamingIngestResponse {
+  abstract Long getStatusCode();
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index 448e773ac..8a84af9c6 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -6,6 +6,7 @@
 
 import static net.snowflake.client.core.Constants.CLOUD_STORAGE_CREDENTIALS_EXPIRED;
 import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CLIENT_CONFIGURE;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
 import static net.snowflake.ingest.utils.Constants.CLIENT_CONFIGURE_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
 import static net.snowflake.ingest.utils.HttpUtil.generateProxyPropertiesForJDBC;
@@ -20,6 +21,7 @@
 import java.util.Optional;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import net.snowflake.client.core.OCSPMode;
 import net.snowflake.client.jdbc.SnowflakeFileTransferAgent;
 import net.snowflake.client.jdbc.SnowflakeFileTransferConfig;
@@ -32,7 +34,6 @@
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
 import net.snowflake.ingest.connection.IngestResponseException;
 import net.snowflake.ingest.connection.RequestBuilder;
-import net.snowflake.ingest.connection.ServiceResponseHandler;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.Utils;
@@ -282,6 +283,20 @@ SnowflakeFileTransferMetadataV1 fetchSignedURL(String fileName)
     return metadata;
   }
 
+  private static class MapStatusGetter<T> implements Function<T, Long> {
+    public MapStatusGetter() {}
+
+    public Long apply(T input) {
+      try {
+        return ((Integer) ((Map<String, Object>) input).get("status_code")).longValue();
+      } catch (Exception e) {
+        throw new SFException(ErrorCode.INTERNAL_ERROR, "failed to get status_code from response");
+      }
+    }
+  }
+
+  private static final MapStatusGetter statusGetter = new MapStatusGetter();
+
   private JsonNode parseClientConfigureResponse(Map<String, Object> response) {
     JsonNode responseNode = mapper.valueToTree(response);
 
@@ -300,13 +315,17 @@ private JsonNode parseClientConfigureResponse(Map<String, Object> response) {
   private Map<String, Object> makeClientConfigureCall(Map<Object, Object> payload)
       throws IOException {
     try {
+
       Map<String, Object> response =
-          ServiceResponseHandler.unmarshallStreamingIngestResponse(
-              httpClient.execute(
-                  requestBuilder.generateStreamingIngestPostRequest(
-                      payload, CLIENT_CONFIGURE_ENDPOINT, "client configure")),
+          executeWithRetries(
               Map.class,
-              STREAMING_CLIENT_CONFIGURE);
+              CLIENT_CONFIGURE_ENDPOINT,
+              mapper.writeValueAsString(payload),
+              "client configure",
+              STREAMING_CLIENT_CONFIGURE,
+              httpClient,
+              requestBuilder,
+              statusGetter);
 
       // Check for Snowflake specific response code
       if (!response.get("status_code").equals((int) RESPONSE_SUCCESS)) {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
new file mode 100644
index 000000000..0236d7284
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
@@ -0,0 +1,118 @@
+package net.snowflake.ingest.streaming.internal;
+
+import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Function;
+import net.snowflake.ingest.connection.IngestResponseException;
+import net.snowflake.ingest.connection.RequestBuilder;
+import net.snowflake.ingest.connection.ServiceResponseHandler;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.http.client.HttpClient;
+
+public class StreamingIngestUtils {
+
+  private static class DefaultStatusGetter<T extends StreamingIngestResponse>
+      implements Function<T, Long> {
+    public DefaultStatusGetter() {}
+
+    public Long apply(T input) {
+      return input.getStatusCode();
+    }
+  }
+
+  private static final DefaultStatusGetter defaultStatusGetter = new DefaultStatusGetter();
+
+  private static final Logging LOGGER = new Logging(StreamingIngestUtils.class);
+
+  private static final ObjectMapper objectMapper = new ObjectMapper();
+
+  static void sleepForRetry(int executionCount) {
+    try {
+      Thread.sleep((1 << (executionCount + 1)) * 1000);
+    } catch (InterruptedException e) {
+      throw new SFException(ErrorCode.INTERNAL_ERROR, e.getMessage());
+    }
+  }
+
+  static <T extends StreamingIngestResponse> T executeWithRetries(
+      Class<T> targetClass,
+      String endpoint,
+      Map<Object, Object> payload,
+      String message,
+      ServiceResponseHandler.ApiName apiName,
+      HttpClient httpClient,
+      RequestBuilder requestBuilder)
+      throws IOException, IngestResponseException {
+    String payloadInString;
+    try {
+      payloadInString = objectMapper.writeValueAsString(payload);
+    } catch (JsonProcessingException e) {
+      throw new SFException(e, ErrorCode.BUILD_REQUEST_FAILURE, message);
+    }
+    return executeWithRetries(
+        targetClass, endpoint, payloadInString, message, apiName, httpClient, requestBuilder);
+  }
+
+  static <T extends StreamingIngestResponse> T executeWithRetries(
+      Class<T> targetClass,
+      String endpoint,
+      String payload,
+      String message,
+      ServiceResponseHandler.ApiName apiName,
+      HttpClient httpClient,
+      RequestBuilder requestBuilder)
+      throws IOException, IngestResponseException {
+    return (T)
+        executeWithRetries(
+            targetClass,
+            endpoint,
+            payload,
+            message,
+            apiName,
+            httpClient,
+            requestBuilder,
+            defaultStatusGetter);
+  }
+
+  static <T> T executeWithRetries(
+      Class<T> targetClass,
+      String endpoint,
+      String payload,
+      String message,
+      ServiceResponseHandler.ApiName apiName,
+      HttpClient httpClient,
+      RequestBuilder requestBuilder,
+      Function<T, Long> statusGetter)
+      throws IOException, IngestResponseException {
+    int retries = 0;
+    T response;
+    do {
+      response =
+          ServiceResponseHandler.unmarshallStreamingIngestResponse(
+              httpClient.execute(
+                  requestBuilder.generateStreamingIngestPostRequest(payload, endpoint, message)),
+              targetClass,
+              apiName);
+
+      if (statusGetter.apply(response) == RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST) {
+        LOGGER.logDebug(
+            "Retrying request for streaming ingest, endpoint={}, retryCount={}, responseCode={}",
+            endpoint,
+            retries,
+            statusGetter.apply(response));
+        retries++;
+        sleepForRetry(retries);
+      } else {
+        return response;
+      }
+    } while (retries <= MAX_STREAMING_INGEST_API_CHANNEL_RETRY);
+    return response;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index 41307d6e8..d7edf88b8 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -24,6 +24,10 @@ public class Constants {
   public static final String PRIVATE_KEY_PASSPHRASE = "private_key_passphrase";
   public static final String JDBC_PRIVATE_KEY = "privateKey";
   public static final long RESPONSE_SUCCESS = 0L; // Don't change, should match server side
+  public static final long RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST =
+      10L; // Don't change, should match server side
+  public static final long RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL =
+      7L; // Don't change, should match server side
   public static final long BLOB_UPLOAD_TIMEOUT_IN_SEC = 5L;
   public static final int BLOB_UPLOAD_MAX_RETRY_COUNT = 12;
   public static final int INSERT_THROTTLE_MAX_RETRY_COUNT = 10;
@@ -46,6 +50,7 @@ public class Constants {
       26; // Don't change, should match server side
   public static final String ENCRYPTION_ALGORITHM = "AES/CTR/NoPadding";
   public static final int ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES = 16;
+  public static final int MAX_STREAMING_INGEST_API_CHANNEL_RETRY = 3;
 
   // Channel level constants
   public static final String CHANNEL_STATUS_ENDPOINT = "/v1/streaming/channels/status/";
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index eddaa9ce0..c2ce76a6b 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -3,12 +3,17 @@
 import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
 import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
+import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
 import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
 import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
 import static net.snowflake.ingest.utils.Constants.ROLE;
 import static net.snowflake.ingest.utils.Constants.USER;
 import static net.snowflake.ingest.utils.ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS_MAP_KEY;
 
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
 import java.io.StringWriter;
@@ -18,11 +23,15 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
 import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils;
+import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.asn1.nist.NISTObjectIdentifiers;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.jcajce.JcaPEMWriter;
@@ -36,6 +45,7 @@
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
 import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Pair;
 import net.snowflake.ingest.utils.ParameterProvider;
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.SnowflakeURL;
@@ -47,6 +57,7 @@
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpPost;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -54,6 +65,73 @@
 public class SnowflakeStreamingIngestClientTest {
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
+  SnowflakeStreamingIngestChannelInternal channel1;
+  SnowflakeStreamingIngestChannelInternal channel2;
+  SnowflakeStreamingIngestChannelInternal channel3;
+  SnowflakeStreamingIngestChannelInternal channel4;
+
+  @Before
+  public void setup() {
+    objectMapper.setVisibility(PropertyAccessor.GETTER, JsonAutoDetect.Visibility.ANY);
+    objectMapper.setVisibility(PropertyAccessor.SETTER, JsonAutoDetect.Visibility.ANY);
+    channel1 =
+        new SnowflakeStreamingIngestChannelInternal(
+            "channel1",
+            "db",
+            "schemaName",
+            "tableName",
+            "0",
+            0L,
+            0L,
+            null,
+            "key",
+            1234L,
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            true);
+    channel2 =
+        new SnowflakeStreamingIngestChannelInternal(
+            "channel2",
+            "db",
+            "schemaName",
+            "tableName",
+            "0",
+            2L,
+            0L,
+            null,
+            "key",
+            1234L,
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            true);
+    channel3 =
+        new SnowflakeStreamingIngestChannelInternal(
+            "channel3",
+            "db",
+            "schemaName",
+            "tableName3",
+            "0",
+            3L,
+            0L,
+            null,
+            "key",
+            1234L,
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            true);
+    channel4 =
+        new SnowflakeStreamingIngestChannelInternal(
+            "channel4",
+            "db",
+            "schemaName",
+            "tableName4",
+            "0",
+            3L,
+            0L,
+            null,
+            "key",
+            1234L,
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            true);
+  }
+
   @Test
   @Ignore // Until able to test in PROD
   public void testConstructorParameters() throws Exception {
@@ -409,6 +487,135 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
         request.getFirstHeader(RequestBuilder.SF_HEADER_AUTHORIZATION_TOKEN_TYPE).getValue());
   }
 
+  private Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> getRetryBlobMetadata() {
+    Map<String, RowBufferStats> columnEps = new HashMap<>();
+    columnEps.put("column", new RowBufferStats());
+    EpInfo epInfo = ArrowRowBuffer.buildEpInfoFromStats(1, columnEps);
+
+    ChannelMetadata channelMetadata1 =
+        ChannelMetadata.builder()
+            .setOwningChannel(channel1)
+            .setRowSequencer(channel1.incrementAndGetRowSequencer())
+            .setOffsetToken(channel1.getOffsetToken())
+            .build();
+    ChannelMetadata channelMetadata2 =
+        ChannelMetadata.builder()
+            .setOwningChannel(channel2)
+            .setRowSequencer(channel2.incrementAndGetRowSequencer())
+            .setOffsetToken(channel2.getOffsetToken())
+            .build();
+    ChannelMetadata channelMetadata3 =
+        ChannelMetadata.builder()
+            .setOwningChannel(channel3)
+            .setRowSequencer(channel3.incrementAndGetRowSequencer())
+            .setOffsetToken(channel3.getOffsetToken())
+            .build();
+    ChannelMetadata channelMetadata4 =
+        ChannelMetadata.builder()
+            .setOwningChannel(channel4)
+            .setRowSequencer(channel4.incrementAndGetRowSequencer())
+            .setOffsetToken(channel4.getOffsetToken())
+            .build();
+
+    List<BlobMetadata> blobs = new ArrayList<>();
+    List<ChunkMetadata> chunks1 = new ArrayList<>();
+    List<ChunkMetadata> chunks2 = new ArrayList<>();
+
+    List<ChannelMetadata> channels1 = new ArrayList<>();
+    channels1.add(channelMetadata1);
+    channels1.add(channelMetadata2);
+    ChunkMetadata chunkMetadata1 =
+        ChunkMetadata.builder()
+            .setOwningTable(channel1)
+            .setChunkStartOffset(0L)
+            .setChunkLength(100)
+            .setChannelList(channels1)
+            .setChunkMD5("md51")
+            .setEncryptionKeyId(1234L)
+            .setEpInfo(epInfo)
+            .build();
+    ChunkMetadata chunkMetadata2 =
+        ChunkMetadata.builder()
+            .setOwningTable(channel2)
+            .setChunkStartOffset(0L)
+            .setChunkLength(100)
+            .setChannelList(Collections.singletonList(channelMetadata3))
+            .setChunkMD5("md52")
+            .setEncryptionKeyId(1234L)
+            .setEpInfo(epInfo)
+            .build();
+    ChunkMetadata chunkMetadata3 =
+        ChunkMetadata.builder()
+            .setOwningTable(channel3)
+            .setChunkStartOffset(0L)
+            .setChunkLength(100)
+            .setChannelList(Collections.singletonList(channelMetadata4))
+            .setChunkMD5("md53")
+            .setEncryptionKeyId(1234L)
+            .setEpInfo(epInfo)
+            .build();
+
+    chunks1.add(chunkMetadata1);
+    chunks1.add(chunkMetadata2);
+    chunks2.add(chunkMetadata3);
+    blobs.add(new BlobMetadata("path1", "md51", chunks1));
+    blobs.add(new BlobMetadata("path2", "md52", chunks2));
+
+    List<ChannelRegisterStatus> channelRegisterStatuses = new ArrayList<>();
+    ChannelRegisterStatus status1 = new ChannelRegisterStatus();
+    status1.setStatusCode(RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL);
+    status1.setChannelName(channelMetadata1.getChannelName());
+    status1.setChannelSequencer(channelMetadata1.getClientSequencer());
+
+    ChannelRegisterStatus status2 = new ChannelRegisterStatus();
+    status2.setStatusCode(RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL);
+    status2.setChannelName(channelMetadata2.getChannelName());
+    status2.setChannelSequencer(channelMetadata2.getClientSequencer());
+
+    channelRegisterStatuses.add(status1);
+    channelRegisterStatuses.add(status2);
+
+    Set<ChunkRegisterStatus> badChunks = new HashSet<>();
+    ChunkRegisterStatus badChunkRegisterStatus = new ChunkRegisterStatus();
+    badChunkRegisterStatus.setDBName(chunkMetadata1.getDBName());
+    badChunkRegisterStatus.setSchemaName(chunkMetadata1.getSchemaName());
+    badChunkRegisterStatus.setTableName(chunkMetadata1.getTableName());
+    badChunkRegisterStatus.setChannelsStatus(channelRegisterStatuses);
+    badChunks.add(badChunkRegisterStatus);
+    return new Pair(blobs, badChunks);
+  }
+
+  @Test
+  public void testGetRetryBlobs() throws Exception {
+    HttpClient httpClient = Mockito.mock(HttpClient.class);
+    RequestBuilder requestBuilder =
+        new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair());
+
+    SnowflakeStreamingIngestClientInternal client =
+        new SnowflakeStreamingIngestClientInternal(
+            "client",
+            new SnowflakeURL("snowflake.dev.local:8082"),
+            null,
+            httpClient,
+            true,
+            requestBuilder,
+            null);
+    Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> testData = getRetryBlobMetadata();
+    List<BlobMetadata> blobs = testData.getFirst();
+    Set<ChunkRegisterStatus> badChunks = testData.getSecond();
+    List<BlobMetadata> result = client.getRetryBlobs(badChunks, blobs);
+    Assert.assertEquals(1, result.size());
+    Assert.assertEquals("path1", result.get(0).getPath());
+    Assert.assertEquals("md51", result.get(0).getMD5());
+    Assert.assertEquals(1, result.get(0).getChunks().size());
+    Assert.assertEquals(2, result.get(0).getChunks().get(0).getChannels().size());
+    Assert.assertEquals(
+        Sets.newHashSet("channel1", "channel2"),
+        result.get(0).getChunks().get(0).getChannels().stream()
+            .map(c -> c.getChannelName())
+            .collect(Collectors.toSet()));
+  }
+
   @Test
   public void testRegisterBlobErrorResponse() throws Exception {
     HttpClient httpClient = Mockito.mock(HttpClient.class);
@@ -544,6 +751,212 @@ public void testRegisterBlobSuccessResponse() throws Exception {
     client.registerBlobs(blobs);
   }
 
+  @Test
+  public void testRegisterBlobsRetries() throws Exception {
+    Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> testData = getRetryBlobMetadata();
+    List<BlobMetadata> blobs = testData.getFirst();
+    Set<ChunkRegisterStatus> badChunks = testData.getSecond();
+
+    ChunkRegisterStatus goodChunkRegisterStatus = new ChunkRegisterStatus();
+    goodChunkRegisterStatus.setDBName(blobs.get(0).getChunks().get(0).getDBName());
+    goodChunkRegisterStatus.setSchemaName(blobs.get(0).getChunks().get(0).getSchemaName());
+    goodChunkRegisterStatus.setTableName(blobs.get(0).getChunks().get(0).getTableName());
+    ChannelRegisterStatus goodStatus1 = new ChannelRegisterStatus();
+    goodStatus1.setStatusCode(RESPONSE_SUCCESS);
+    goodStatus1.setChannelName("channel3");
+    goodStatus1.setChannelSequencer(3L);
+    goodChunkRegisterStatus.setChannelsStatus(Collections.singletonList(goodStatus1));
+
+    ChunkRegisterStatus goodChunkRegisterStatus2 = new ChunkRegisterStatus();
+    goodChunkRegisterStatus2.setDBName(blobs.get(0).getChunks().get(0).getDBName());
+    goodChunkRegisterStatus2.setSchemaName(blobs.get(0).getChunks().get(0).getSchemaName());
+    goodChunkRegisterStatus2.setTableName(blobs.get(0).getChunks().get(0).getTableName());
+    ChannelRegisterStatus goodStatus2 = new ChannelRegisterStatus();
+    goodStatus2.setStatusCode(RESPONSE_SUCCESS);
+    goodStatus2.setChannelName("channel4");
+    goodStatus2.setChannelSequencer(3L);
+    goodChunkRegisterStatus2.setChannelsStatus(Collections.singletonList(goodStatus2));
+
+    RegisterBlobResponse initialResponse = new RegisterBlobResponse();
+    initialResponse.setMessage("successish");
+    initialResponse.setStatusCode(RESPONSE_SUCCESS);
+
+    RegisterBlobResponse retryResponse = new RegisterBlobResponse();
+    retryResponse.setMessage("successish");
+    retryResponse.setStatusCode(RESPONSE_SUCCESS);
+
+    List<BlobRegisterStatus> blobRegisterStatuses = new ArrayList<>();
+    BlobRegisterStatus blobRegisterStatus1 = new BlobRegisterStatus();
+    blobRegisterStatus1.setChunksStatus(badChunks.stream().collect(Collectors.toList()));
+    blobRegisterStatuses.add(blobRegisterStatus1);
+    BlobRegisterStatus blobRegisterStatus2 = new BlobRegisterStatus();
+    blobRegisterStatus2.setChunksStatus(Collections.singletonList(goodChunkRegisterStatus));
+    blobRegisterStatuses.add(blobRegisterStatus2);
+    BlobRegisterStatus blobRegisterStatus3 = new BlobRegisterStatus();
+    blobRegisterStatus3.setChunksStatus(Collections.singletonList(goodChunkRegisterStatus2));
+    blobRegisterStatuses.add(blobRegisterStatus3);
+    initialResponse.setBlobsStatus(blobRegisterStatuses);
+
+    retryResponse.setBlobsStatus(Collections.singletonList(blobRegisterStatus1));
+
+    String responseString = objectMapper.writeValueAsString(initialResponse);
+    String retryResponseString = objectMapper.writeValueAsString(retryResponse);
+
+    HttpClient httpClient = Mockito.mock(HttpClient.class);
+    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    StatusLine statusLine = Mockito.mock(StatusLine.class);
+    HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
+    Mockito.when(statusLine.getStatusCode()).thenReturn(200);
+    Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    Mockito.when(httpResponse.getEntity()).thenReturn(httpEntity);
+    Mockito.when(httpEntity.getContent())
+        .thenReturn(
+            IOUtils.toInputStream(responseString),
+            IOUtils.toInputStream(retryResponseString),
+            IOUtils.toInputStream(retryResponseString),
+            IOUtils.toInputStream(retryResponseString));
+    Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
+
+    RequestBuilder requestBuilder =
+        Mockito.spy(
+            new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair()));
+    SnowflakeStreamingIngestClientInternal client =
+        new SnowflakeStreamingIngestClientInternal(
+            "client",
+            new SnowflakeURL("snowflake.dev.local:8082"),
+            null,
+            httpClient,
+            true,
+            requestBuilder,
+            null);
+
+    client.getChannelCache().addChannel(channel1);
+    client.getChannelCache().addChannel(channel2);
+    client.getChannelCache().addChannel(channel3);
+    client.getChannelCache().addChannel(channel4);
+    client.registerBlobs(blobs);
+    Mockito.verify(requestBuilder, Mockito.times(MAX_STREAMING_INGEST_API_CHANNEL_RETRY + 1))
+        .generateStreamingIngestPostRequest(Mockito.anyString(), Mockito.any(), Mockito.any());
+    Assert.assertFalse(channel1.isValid());
+    Assert.assertFalse(channel2.isValid());
+  }
+
+  @Test
+  public void testRegisterBlobsRetriesSucceeds() throws Exception {
+    Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> testData = getRetryBlobMetadata();
+    List<BlobMetadata> blobs = testData.getFirst();
+    Set<ChunkRegisterStatus> badChunks = testData.getSecond();
+
+    ChunkRegisterStatus goodChunkRegisterStatus = new ChunkRegisterStatus();
+    goodChunkRegisterStatus.setDBName(blobs.get(0).getChunks().get(0).getDBName());
+    goodChunkRegisterStatus.setSchemaName(blobs.get(0).getChunks().get(0).getSchemaName());
+    goodChunkRegisterStatus.setTableName(blobs.get(0).getChunks().get(0).getTableName());
+    ChannelRegisterStatus goodStatus1 = new ChannelRegisterStatus();
+    goodStatus1.setStatusCode(RESPONSE_SUCCESS);
+    goodStatus1.setChannelName("channel3");
+    goodStatus1.setChannelSequencer(3L);
+    goodChunkRegisterStatus.setChannelsStatus(Collections.singletonList(goodStatus1));
+
+    ChunkRegisterStatus goodChunkRegisterStatus2 = new ChunkRegisterStatus();
+    goodChunkRegisterStatus2.setDBName(blobs.get(0).getChunks().get(0).getDBName());
+    goodChunkRegisterStatus2.setSchemaName(blobs.get(0).getChunks().get(0).getSchemaName());
+    goodChunkRegisterStatus2.setTableName(blobs.get(0).getChunks().get(0).getTableName());
+    ChannelRegisterStatus goodStatus2 = new ChannelRegisterStatus();
+    goodStatus2.setStatusCode(RESPONSE_SUCCESS);
+    goodStatus2.setChannelName("channel4");
+    goodStatus2.setChannelSequencer(3L);
+    goodChunkRegisterStatus2.setChannelsStatus(Collections.singletonList(goodStatus2));
+
+    List<ChunkRegisterStatus> goodChunkRegisterRetryStatus =
+        badChunks.stream()
+            .map(
+                chunkRegisterStatus -> {
+                  ChunkRegisterStatus newStatus = new ChunkRegisterStatus();
+                  newStatus.setTableName(chunkRegisterStatus.getTableName());
+                  newStatus.setDBName(chunkRegisterStatus.getDBName());
+                  newStatus.setSchemaName(chunkRegisterStatus.getSchemaName());
+                  newStatus.setChannelsStatus(
+                      chunkRegisterStatus.getChannelsStatus().stream()
+                          .map(
+                              channelRegisterStatus -> {
+                                ChannelRegisterStatus newChannelStatus =
+                                    new ChannelRegisterStatus();
+                                newChannelStatus.setStatusCode(RESPONSE_SUCCESS);
+                                newChannelStatus.setChannelSequencer(
+                                    channelRegisterStatus.getChannelSequencer());
+                                newChannelStatus.setChannelName(
+                                    channelRegisterStatus.getChannelName());
+                                return newChannelStatus;
+                              })
+                          .collect(Collectors.toList()));
+                  return newStatus;
+                })
+            .collect(Collectors.toList());
+
+    RegisterBlobResponse initialResponse = new RegisterBlobResponse();
+    initialResponse.setMessage("successish");
+    initialResponse.setStatusCode(RESPONSE_SUCCESS);
+
+    RegisterBlobResponse retryResponse = new RegisterBlobResponse();
+    retryResponse.setMessage("successish");
+    retryResponse.setStatusCode(RESPONSE_SUCCESS);
+
+    List<BlobRegisterStatus> blobRegisterStatuses = new ArrayList<>();
+    BlobRegisterStatus blobRegisterStatus1 = new BlobRegisterStatus();
+    blobRegisterStatus1.setChunksStatus(badChunks.stream().collect(Collectors.toList()));
+    blobRegisterStatuses.add(blobRegisterStatus1);
+    BlobRegisterStatus blobRegisterStatus2 = new BlobRegisterStatus();
+    blobRegisterStatus2.setChunksStatus(Collections.singletonList(goodChunkRegisterStatus));
+    blobRegisterStatuses.add(blobRegisterStatus2);
+    BlobRegisterStatus blobRegisterStatus3 = new BlobRegisterStatus();
+    blobRegisterStatus3.setChunksStatus(Collections.singletonList(goodChunkRegisterStatus2));
+    blobRegisterStatuses.add(blobRegisterStatus3);
+    initialResponse.setBlobsStatus(blobRegisterStatuses);
+
+    BlobRegisterStatus retryBlobRegisterStatus = new BlobRegisterStatus();
+    retryBlobRegisterStatus.setChunksStatus(goodChunkRegisterRetryStatus);
+    retryResponse.setBlobsStatus(Collections.singletonList(retryBlobRegisterStatus));
+
+    String responseString = objectMapper.writeValueAsString(initialResponse);
+    String retryResponseString = objectMapper.writeValueAsString(retryResponse);
+
+    HttpClient httpClient = Mockito.mock(HttpClient.class);
+    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    StatusLine statusLine = Mockito.mock(StatusLine.class);
+    HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
+    Mockito.when(statusLine.getStatusCode()).thenReturn(200);
+    Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    Mockito.when(httpResponse.getEntity()).thenReturn(httpEntity);
+    Mockito.when(httpEntity.getContent())
+        .thenReturn(
+            IOUtils.toInputStream(responseString), IOUtils.toInputStream(retryResponseString));
+    Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
+
+    RequestBuilder requestBuilder =
+        Mockito.spy(
+            new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair()));
+    SnowflakeStreamingIngestClientInternal client =
+        new SnowflakeStreamingIngestClientInternal(
+            "client",
+            new SnowflakeURL("snowflake.dev.local:8082"),
+            null,
+            httpClient,
+            true,
+            requestBuilder,
+            null);
+
+    client.getChannelCache().addChannel(channel1);
+    client.getChannelCache().addChannel(channel2);
+    client.getChannelCache().addChannel(channel3);
+    client.getChannelCache().addChannel(channel4);
+
+    client.registerBlobs(blobs);
+    Mockito.verify(requestBuilder, Mockito.times(2))
+        .generateStreamingIngestPostRequest(Mockito.anyString(), Mockito.any(), Mockito.any());
+    Assert.assertTrue(channel1.isValid());
+    Assert.assertTrue(channel2.isValid());
+  }
+
   @Test
   public void testRegisterBlobResponseWithInvalidChannel() throws Exception {
     String dbName = "DB_STREAMINGINGEST";
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
index 0aecce7e0..6b40e3dc0 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
@@ -16,7 +16,6 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Properties;
 import java.util.concurrent.ExecutorService;
@@ -268,10 +267,10 @@ public void testRefreshSnowflakeMetadataRemote() throws Exception {
         stage.refreshSnowflakeMetadata(true);
 
     final ArgumentCaptor<String> endpointCaptor = ArgumentCaptor.forClass(String.class);
-    final ArgumentCaptor<Map<Object, Object>> mapCaptor = ArgumentCaptor.forClass(Map.class);
+    final ArgumentCaptor<String> stringCaptor = ArgumentCaptor.forClass(String.class);
     Mockito.verify(mockBuilder)
         .generateStreamingIngestPostRequest(
-            mapCaptor.capture(), endpointCaptor.capture(), Mockito.any());
+            stringCaptor.capture(), endpointCaptor.capture(), Mockito.any());
     Assert.assertEquals(Constants.CLIENT_CONFIGURE_ENDPOINT, endpointCaptor.getValue());
     Assert.assertTrue(metadataWithAge.timestamp.isPresent());
     Assert.assertEquals(
@@ -304,10 +303,10 @@ public void testFetchSignedURL() throws Exception {
     SnowflakeFileTransferMetadataV1 metadata = stage.fetchSignedURL("path/fileName");
 
     final ArgumentCaptor<String> endpointCaptor = ArgumentCaptor.forClass(String.class);
-    final ArgumentCaptor<Map<Object, Object>> mapCaptor = ArgumentCaptor.forClass(Map.class);
+    final ArgumentCaptor<String> stringCaptor = ArgumentCaptor.forClass(String.class);
     Mockito.verify(mockBuilder)
         .generateStreamingIngestPostRequest(
-            mapCaptor.capture(), endpointCaptor.capture(), Mockito.any());
+            stringCaptor.capture(), endpointCaptor.capture(), Mockito.any());
     Assert.assertEquals(Constants.CLIENT_CONFIGURE_ENDPOINT, endpointCaptor.getValue());
     Assert.assertEquals(StageInfo.StageType.S3, metadata.getStageInfo().getStageType());
     Assert.assertEquals("foo/streaming_ingest/", metadata.getStageInfo().getLocation());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
new file mode 100644
index 000000000..314235aa4
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
@@ -0,0 +1,165 @@
+package net.snowflake.ingest.streaming.internal;
+
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CHANNEL_STATUS;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
+import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.InputStream;
+import java.util.ArrayList;
+import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils;
+import net.snowflake.ingest.TestUtils;
+import net.snowflake.ingest.connection.RequestBuilder;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.StatusLine;
+import org.apache.http.client.HttpClient;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+public class StreamingIngestUtilsTest {
+  private static final ObjectMapper objectMapper = new ObjectMapper();
+
+  @Test
+  public void testSuccessNoRetries() throws Exception {
+    ChannelsStatusResponse response = new ChannelsStatusResponse();
+    response.setStatusCode(0L);
+    response.setMessage("honk");
+    response.setChannels(new ArrayList<>());
+    String responseString = objectMapper.writeValueAsString(response);
+
+    HttpClient httpClient = Mockito.mock(HttpClient.class);
+    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    StatusLine statusLine = Mockito.mock(StatusLine.class);
+    HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
+    Mockito.when(statusLine.getStatusCode()).thenReturn(200);
+    Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    Mockito.when(httpResponse.getEntity()).thenReturn(httpEntity);
+    Mockito.when(httpEntity.getContent()).thenReturn(IOUtils.toInputStream(responseString));
+    Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
+
+    RequestBuilder requestBuilder =
+        Mockito.spy(
+            new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair()));
+
+    executeWithRetries(
+        ChannelsStatusResponse.class,
+        CHANNEL_STATUS_ENDPOINT,
+        "{}",
+        "channel status",
+        STREAMING_CHANNEL_STATUS,
+        httpClient,
+        requestBuilder);
+
+    Mockito.verify(requestBuilder, Mockito.times(1))
+        .generateStreamingIngestPostRequest(Mockito.anyString(), Mockito.any(), Mockito.any());
+  }
+
+  InputStream getInputStream(String value) {
+    return IOUtils.toInputStream(value);
+  }
+
+  @Test
+  public void testRetries() throws Exception {
+    ChannelsStatusResponse response = new ChannelsStatusResponse();
+    response.setStatusCode(RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST);
+    //    response.setStatusCode(7L);
+
+    response.setMessage("honk");
+    response.setChannels(new ArrayList<>());
+    String responseString = objectMapper.writeValueAsString(response);
+
+    HttpClient httpClient = Mockito.mock(HttpClient.class);
+    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    StatusLine statusLine = Mockito.mock(StatusLine.class);
+    HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
+    Mockito.when(statusLine.getStatusCode()).thenReturn(200);
+    Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    Mockito.when(httpResponse.getEntity()).thenReturn(httpEntity);
+
+    Mockito.when(httpEntity.getContent())
+        .thenAnswer(
+            new Answer<InputStream>() {
+              @Override
+              public InputStream answer(InvocationOnMock invocation) throws Throwable {
+                return IOUtils.toInputStream(responseString);
+              }
+            });
+
+    Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
+
+    RequestBuilder requestBuilder =
+        Mockito.spy(
+            new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair()));
+
+    ChannelsStatusResponse result =
+        executeWithRetries(
+            ChannelsStatusResponse.class,
+            CHANNEL_STATUS_ENDPOINT,
+            "{}",
+            "channel status",
+            STREAMING_CHANNEL_STATUS,
+            httpClient,
+            requestBuilder);
+
+    Mockito.verify(requestBuilder, Mockito.times(4))
+        .generateStreamingIngestPostRequest(Mockito.anyString(), Mockito.any(), Mockito.any());
+    Assert.assertEquals("honk", result.getMessage());
+  }
+
+  @Test
+  public void testRetriesRecovery() throws Exception {
+    ChannelsStatusResponse errorResponse = new ChannelsStatusResponse();
+    errorResponse.setStatusCode(RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST);
+
+    errorResponse.setMessage("honkFailure");
+    errorResponse.setChannels(new ArrayList<>());
+    String errorResponseString = objectMapper.writeValueAsString(errorResponse);
+
+    ChannelsStatusResponse successfulResponse = new ChannelsStatusResponse();
+    successfulResponse.setStatusCode(RESPONSE_SUCCESS);
+
+    successfulResponse.setMessage("honkSuccess");
+    successfulResponse.setChannels(new ArrayList<>());
+    String successfulResponseString = objectMapper.writeValueAsString(successfulResponse);
+
+    HttpClient httpClient = Mockito.mock(HttpClient.class);
+    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    StatusLine statusLine = Mockito.mock(StatusLine.class);
+    HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
+    Mockito.when(statusLine.getStatusCode()).thenReturn(200);
+    Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    Mockito.when(httpResponse.getEntity()).thenReturn(httpEntity);
+
+    Mockito.when(httpEntity.getContent())
+        .thenReturn(
+            IOUtils.toInputStream(errorResponseString),
+            IOUtils.toInputStream(errorResponseString),
+            IOUtils.toInputStream(successfulResponseString));
+    Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
+
+    RequestBuilder requestBuilder =
+        Mockito.spy(
+            new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair()));
+
+    ChannelsStatusResponse result =
+        executeWithRetries(
+            ChannelsStatusResponse.class,
+            CHANNEL_STATUS_ENDPOINT,
+            "{}",
+            "channel status",
+            STREAMING_CHANNEL_STATUS,
+            httpClient,
+            requestBuilder);
+
+    Mockito.verify(requestBuilder, Mockito.times(3))
+        .generateStreamingIngestPostRequest(Mockito.anyString(), Mockito.any(), Mockito.any());
+
+    Assert.assertEquals("honkSuccess", result.getMessage());
+  }
+}

From 53461a8d405a82a7ce53269bc99c1b6a3acb4db9 Mon Sep 17 00:00:00 2001
From: Jay Patel <jay.patel@snowflake.com>
Date: Thu, 21 Apr 2022 14:10:08 -0700
Subject: [PATCH 019/356] Use CloseableHttpClient and consume Entity on all
 APIs (#167)

* BUG_FIX Consume the response entity to free up connection (#164)

* Consume the response entity to free up connection

- As mentioned in httpclient docs
https://hc.apache.org/httpcomponents-client-4.5.x/quickstart.html
Please note that if response content is not fully consumed the underlying
connection cannot be safely re-used and will be shut down and discarded
by the connection manager.

* Helper static function for consuming entity

* Replace httpClient with CloseableHttpClient to close the response in try with resource

* Make changes for streaming ingest http calls
---
 .../snowflake/ingest/SimpleIngestManager.java | 56 ++++++++++---------
 .../connection/ServiceResponseHandler.java    | 40 +++++++++----
 ...nowflakeStreamingIngestClientInternal.java |  8 +--
 .../internal/StreamingIngestStage.java        |  8 +--
 .../internal/StreamingIngestUtils.java        | 22 ++++----
 .../net/snowflake/ingest/utils/HttpUtil.java  |  6 +-
 .../SnowflakeStreamingIngestChannelTest.java  | 16 +++---
 .../SnowflakeStreamingIngestClientTest.java   | 38 ++++++-------
 .../internal/StreamingIngestStageTest.java    | 16 +++---
 .../internal/StreamingIngestUtilsTest.java    | 16 +++---
 10 files changed, 124 insertions(+), 102 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
index 1064c25ab..2ccfea05c 100644
--- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
+++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
@@ -32,10 +32,10 @@
 import net.snowflake.ingest.utils.HttpUtil;
 import net.snowflake.ingest.utils.StagedFileWrapper;
 import net.snowflake.ingest.utils.Utils;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -197,7 +197,7 @@ public SimpleIngestManager build() {
   // logger object for this class
   private static final Logger LOGGER = LoggerFactory.getLogger(SimpleIngestManager.class);
   // HTTP Client that we use for sending requests to the service
-  private HttpClient httpClient;
+  private CloseableHttpClient httpClient;
 
   // the account in which the user lives
   private String account;
@@ -564,13 +564,13 @@ public IngestResponse ingestFiles(
             requestId, pipe, files, showSkippedFiles, Optional.ofNullable(clientInfo));
 
     // send the request and get a response....
-    HttpResponse response = httpClient.execute(httpPostForIngestFile);
-
-    LOGGER.info(
-        "Attempting to unmarshall insert response - {}, with clientInfo - {}",
-        response,
-        clientInfo);
-    return ServiceResponseHandler.unmarshallIngestResponse(response, requestId);
+    try (CloseableHttpResponse response = httpClient.execute(httpPostForIngestFile)) {
+      LOGGER.info(
+          "Attempting to unmarshall insert response - {}, with clientInfo - {}",
+          response,
+          clientInfo);
+      return ServiceResponseHandler.unmarshallIngestResponse(response, requestId);
+    }
   }
 
   /**
@@ -597,10 +597,10 @@ public HistoryResponse getHistory(UUID requestId, Integer recentSeconds, String
         builder.generateHistoryRequest(requestId, pipe, recentSeconds, beginMark);
 
     // send the request and get a response...
-    HttpResponse response = httpClient.execute(httpGetHistory);
-
-    LOGGER.info("Attempting to unmarshall history response - {}", response);
-    return ServiceResponseHandler.unmarshallHistoryResponse(response, requestId);
+    try (CloseableHttpResponse response = httpClient.execute(httpGetHistory)) {
+      LOGGER.info("Attempting to unmarshall history response - {}", response);
+      return ServiceResponseHandler.unmarshallHistoryResponse(response, requestId);
+    }
   }
 
   /**
@@ -626,13 +626,13 @@ public HistoryRangeResponse getHistoryRange(
       requestId = UUID.randomUUID();
     }
 
-    HttpResponse response =
+    try (CloseableHttpResponse response =
         httpClient.execute(
             builder.generateHistoryRangeRequest(
-                requestId, pipe, startTimeInclusive, endTimeExclusive));
-
-    LOGGER.info("Attempting to unmarshall history range response - {}", response);
-    return ServiceResponseHandler.unmarshallHistoryRangeResponse(response, requestId);
+                requestId, pipe, startTimeInclusive, endTimeExclusive))) {
+      LOGGER.info("Attempting to unmarshall history range response - {}", response);
+      return ServiceResponseHandler.unmarshallHistoryRangeResponse(response, requestId);
+    }
   }
 
   /**
@@ -651,10 +651,11 @@ public ConfigureClientResponse configureClient(UUID requestId)
     if (requestId == null || requestId.toString().isEmpty()) {
       requestId = UUID.randomUUID();
     }
-    HttpResponse response =
-        httpClient.execute(builder.generateConfigureClientRequest(requestId, pipe));
-    LOGGER.info("Attempting to unmarshall configure client response - {}", response);
-    return ServiceResponseHandler.unmarshallConfigureClientResponse(response, requestId);
+    try (CloseableHttpResponse response =
+        httpClient.execute(builder.generateConfigureClientRequest(requestId, pipe))) {
+      LOGGER.info("Attempting to unmarshall configure client response - {}", response);
+      return ServiceResponseHandler.unmarshallConfigureClientResponse(response, requestId);
+    }
   }
 
   /**
@@ -673,10 +674,11 @@ public ClientStatusResponse getClientStatus(UUID requestId)
     if (requestId == null || requestId.toString().isEmpty()) {
       requestId = UUID.randomUUID();
     }
-    HttpResponse response =
-        httpClient.execute(builder.generateGetClientStatusRequest(requestId, pipe));
-    LOGGER.info("Attempting to unmarshall get client status response - {}", response);
-    return ServiceResponseHandler.unmarshallGetClientStatus(response, requestId);
+    try (CloseableHttpResponse response =
+        httpClient.execute(builder.generateGetClientStatusRequest(requestId, pipe))) {
+      LOGGER.info("Attempting to unmarshall get client status response - {}", response);
+      return ServiceResponseHandler.unmarshallGetClientStatus(response, requestId);
+    }
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java
index 8c9590a22..ea3ae715e 100644
--- a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java
+++ b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java
@@ -9,6 +9,7 @@
 import java.io.IOException;
 import java.util.UUID;
 import net.snowflake.ingest.utils.BackOffException;
+import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
 import org.apache.http.StatusLine;
@@ -91,8 +92,7 @@ public static IngestResponse unmarshallIngestResponse(HttpResponse response, UUI
     // handle the exceptional status code
     handleExceptionalStatus(response, requestId, ApiName.INSERT_FILES);
 
-    // grab the response entity
-    String blob = EntityUtils.toString(response.getEntity());
+    String blob = consumeAndReturnResponseEntityAsString(response.getEntity());
 
     // Read out the blob entity into a class
     return mapper.readValue(blob, IngestResponse.class);
@@ -120,8 +120,7 @@ public static HistoryResponse unmarshallHistoryResponse(HttpResponse response, U
     // handle the exceptional status code
     handleExceptionalStatus(response, requestId, ApiName.INSERT_REPORT);
 
-    // grab the string version of the response entity
-    String blob = EntityUtils.toString(response.getEntity());
+    String blob = consumeAndReturnResponseEntityAsString(response.getEntity());
 
     // read out our blob into a pojo
     return mapper.readValue(blob, HistoryResponse.class);
@@ -150,9 +149,7 @@ public static HistoryRangeResponse unmarshallHistoryRangeResponse(
     // handle the exceptional status code
     handleExceptionalStatus(response, requestId, ApiName.LOAD_HISTORY_SCAN);
 
-    // grab the string version of the response entity
-    String blob = EntityUtils.toString(response.getEntity());
-
+    String blob = consumeAndReturnResponseEntityAsString(response.getEntity());
     // read out our blob into a pojo
     return mapper.readValue(blob, HistoryRangeResponse.class);
   }
@@ -180,7 +177,7 @@ public static ConfigureClientResponse unmarshallConfigureClientResponse(
     handleExceptionalStatus(response, requestId, ApiName.CLIENT_CONFIGURE);
 
     // grab the string version of the response entity
-    String blob = EntityUtils.toString(response.getEntity());
+    String blob = consumeAndReturnResponseEntityAsString(response.getEntity());
 
     // read out our blob into a pojo
     return mapper.readValue(blob, ConfigureClientResponse.class);
@@ -209,7 +206,7 @@ public static ClientStatusResponse unmarshallGetClientStatus(
     handleExceptionalStatus(response, requestId, ApiName.CLIENT_STATUS);
 
     // grab the string version of the response entity
-    String blob = EntityUtils.toString(response.getEntity());
+    String blob = consumeAndReturnResponseEntityAsString(response.getEntity());
 
     // read out our blob into a pojo
     return mapper.readValue(blob, ClientStatusResponse.class);
@@ -239,7 +236,7 @@ public static <T> T unmarshallStreamingIngestResponse(
     handleExceptionalStatus(response, null, apiName);
 
     // Grab the string version of the response entity
-    String blob = EntityUtils.toString(response.getEntity());
+    String blob = consumeAndReturnResponseEntityAsString(response.getEntity());
 
     // Read out our blob into a pojo
     return mapper.readValue(blob, valueType);
@@ -274,11 +271,32 @@ private static void handleExceptionalStatus(
               apiName,
               statusLine.getStatusCode(),
               requestId == null ? "" : requestId.toString());
-          String blob = EntityUtils.toString(response.getEntity());
+          String blob = consumeAndReturnResponseEntityAsString(response.getEntity());
           throw new IngestResponseException(
               statusLine.getStatusCode(),
               IngestResponseException.IngestExceptionBody.parseBody(blob));
       }
     }
   }
+
+  /**
+   * Consumes the HttpEntity as mentioned in <a
+   * href="https://hc.apache.org/httpcomponents-client-4.5.x/quickstart.html">HttpClient Docs</a>
+   *
+   * <p>Also returns the string version of this entity.
+   *
+   * @param httpResponseEntity the response entity obtained after successfully calling associated
+   *     Rest APIs
+   * @return String version of this http response which will be later used to deserialize into
+   *     respective Response Object
+   * @throws IOException if parsing error
+   */
+  private static String consumeAndReturnResponseEntityAsString(HttpEntity httpResponseEntity)
+      throws IOException {
+    // grab the string version of the response entity
+    String responseEntityAsString = EntityUtils.toString(httpResponseEntity);
+
+    EntityUtils.consumeQuietly(httpResponseEntity);
+    return responseEntityAsString;
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 7d7a5d374..f4abef7c3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -70,7 +70,7 @@
 import net.snowflake.ingest.utils.Utils;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.http.client.HttpClient;
+import org.apache.http.impl.client.CloseableHttpClient;
 
 /**
  * The first version of implementation for SnowflakeStreamingIngestClient. The client internally
@@ -98,7 +98,7 @@ public class SnowflakeStreamingIngestClientInternal implements SnowflakeStreamin
   private String role;
 
   // Http client to send HTTP request to Snowflake
-  private final HttpClient httpClient;
+  private final CloseableHttpClient httpClient;
 
   // Reference to the channel cache
   private final ChannelCache channelCache;
@@ -148,7 +148,7 @@ public class SnowflakeStreamingIngestClientInternal implements SnowflakeStreamin
       String name,
       SnowflakeURL accountURL,
       Properties prop,
-      HttpClient httpClient,
+      CloseableHttpClient httpClient,
       boolean isTestMode,
       RequestBuilder requestBuilder,
       Map<String, Object> parameterOverrides) {
@@ -596,7 +596,7 @@ boolean isTestMode() {
   }
 
   /** Get the http client */
-  HttpClient getHttpClient() {
+  CloseableHttpClient getHttpClient() {
     return this.httpClient;
   }
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index 8a84af9c6..338c735ad 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -38,7 +38,7 @@
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.Utils;
 import org.apache.arrow.util.VisibleForTesting;
-import org.apache.http.client.HttpClient;
+import org.apache.http.impl.client.CloseableHttpClient;
 
 /** Handles uploading files to the Snowflake Streaming Ingest Stage */
 class StreamingIngestStage {
@@ -77,7 +77,7 @@ state to record unknown age.
   }
 
   private SnowflakeFileTransferMetadataWithAge fileTransferMetadataWithAge;
-  private final HttpClient httpClient;
+  private final CloseableHttpClient httpClient;
   private final RequestBuilder requestBuilder;
   private final String role;
   private final String clientName;
@@ -89,7 +89,7 @@ state to record unknown age.
   StreamingIngestStage(
       boolean isTestMode,
       String role,
-      HttpClient httpClient,
+      CloseableHttpClient httpClient,
       RequestBuilder requestBuilder,
       String clientName)
       throws SnowflakeSQLException, IOException {
@@ -117,7 +117,7 @@ state to record unknown age.
   StreamingIngestStage(
       boolean isTestMode,
       String role,
-      HttpClient httpClient,
+      CloseableHttpClient httpClient,
       RequestBuilder requestBuilder,
       String clientName,
       SnowflakeFileTransferMetadataWithAge testMetadata)
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
index 0236d7284..d0c0f3a96 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
@@ -14,7 +14,8 @@
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.SFException;
-import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.impl.client.CloseableHttpClient;
 
 public class StreamingIngestUtils {
 
@@ -47,7 +48,7 @@ static <T extends StreamingIngestResponse> T executeWithRetries(
       Map<Object, Object> payload,
       String message,
       ServiceResponseHandler.ApiName apiName,
-      HttpClient httpClient,
+      CloseableHttpClient httpClient,
       RequestBuilder requestBuilder)
       throws IOException, IngestResponseException {
     String payloadInString;
@@ -66,7 +67,7 @@ static <T extends StreamingIngestResponse> T executeWithRetries(
       String payload,
       String message,
       ServiceResponseHandler.ApiName apiName,
-      HttpClient httpClient,
+      CloseableHttpClient httpClient,
       RequestBuilder requestBuilder)
       throws IOException, IngestResponseException {
     return (T)
@@ -87,19 +88,20 @@ static <T> T executeWithRetries(
       String payload,
       String message,
       ServiceResponseHandler.ApiName apiName,
-      HttpClient httpClient,
+      CloseableHttpClient httpClient,
       RequestBuilder requestBuilder,
       Function<T, Long> statusGetter)
       throws IOException, IngestResponseException {
     int retries = 0;
     T response;
     do {
-      response =
-          ServiceResponseHandler.unmarshallStreamingIngestResponse(
-              httpClient.execute(
-                  requestBuilder.generateStreamingIngestPostRequest(payload, endpoint, message)),
-              targetClass,
-              apiName);
+      try (CloseableHttpResponse httpResponse =
+          httpClient.execute(
+              requestBuilder.generateStreamingIngestPostRequest(payload, endpoint, message))) {
+        response =
+            ServiceResponseHandler.unmarshallStreamingIngestResponse(
+                httpResponse, targetClass, apiName);
+      }
 
       if (statusGetter.apply(response) == RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST) {
         LOGGER.logDebug(
diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index 8e36e192e..565ffc408 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -19,7 +19,6 @@
 import org.apache.http.auth.Credentials;
 import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.CredentialsProvider;
-import org.apache.http.client.HttpClient;
 import org.apache.http.client.HttpRequestRetryHandler;
 import org.apache.http.client.ServiceUnavailableRetryStrategy;
 import org.apache.http.client.config.RequestConfig;
@@ -27,6 +26,7 @@
 import org.apache.http.conn.ssl.DefaultHostnameVerifier;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.conn.DefaultProxyRoutePlanner;
@@ -48,9 +48,9 @@ public class HttpUtil {
   private static final int MAX_RETRIES = 3;
   static final int DEFAULT_CONNECTION_TIMEOUT = 1; // minute
   static final int DEFAULT_HTTP_CLIENT_SOCKET_TIMEOUT = 5; // minutes
-  private static HttpClient httpClient;
+  private static CloseableHttpClient httpClient;
 
-  public static HttpClient getHttpClient() {
+  public static CloseableHttpClient getHttpClient() {
     if (httpClient == null) {
       initHttpClient();
     }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 563858782..351e2457d 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -30,10 +30,10 @@
 import net.snowflake.ingest.utils.Utils;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHeaders;
-import org.apache.http.HttpResponse;
 import org.apache.http.StatusLine;
-import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -272,8 +272,8 @@ public void testOpenChannelPostRequest() throws Exception {
 
   @Test
   public void testOpenChannelErrorResponse() throws Exception {
-    HttpClient httpClient = Mockito.mock(HttpClient.class);
-    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
     Mockito.when(statusLine.getStatusCode()).thenReturn(500);
@@ -343,8 +343,8 @@ public void testOpenChannelSnowflakeInternalErrorResponse() throws Exception {
             + "  } ]\n"
             + "}";
 
-    HttpClient httpClient = Mockito.mock(HttpClient.class);
-    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
     Mockito.when(statusLine.getStatusCode()).thenReturn(200);
@@ -423,8 +423,8 @@ public void testOpenChannelSuccessResponse() throws Exception {
             + "  \"encryption_key\" : \"3/l6q2xeDurO4ljfde4DXA==\"\n"
             + "}";
 
-    HttpClient httpClient = Mockito.mock(HttpClient.class);
-    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
     Mockito.when(statusLine.getStatusCode()).thenReturn(200);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index c2ce76a6b..cd62eb144 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -52,10 +52,10 @@
 import net.snowflake.ingest.utils.Utils;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHeaders;
-import org.apache.http.HttpResponse;
 import org.apache.http.StatusLine;
-import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -307,8 +307,8 @@ public void testGetChannelsStatusWithRequest() throws Exception {
     response.setChannels(new ArrayList<>());
     String responseString = objectMapper.writeValueAsString(response);
 
-    HttpClient httpClient = Mockito.mock(HttpClient.class);
-    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
     Mockito.when(statusLine.getStatusCode()).thenReturn(200);
@@ -365,8 +365,8 @@ public void testGetChannelsStatusWithRequestError() throws Exception {
     response.setChannels(new ArrayList<>());
     String responseString = objectMapper.writeValueAsString(response);
 
-    HttpClient httpClient = Mockito.mock(HttpClient.class);
-    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
     Mockito.when(statusLine.getStatusCode()).thenReturn(500);
@@ -587,7 +587,7 @@ private Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> getRetryBlobMetadata(
 
   @Test
   public void testGetRetryBlobs() throws Exception {
-    HttpClient httpClient = Mockito.mock(HttpClient.class);
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
     RequestBuilder requestBuilder =
         new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair());
 
@@ -618,8 +618,8 @@ public void testGetRetryBlobs() throws Exception {
 
   @Test
   public void testRegisterBlobErrorResponse() throws Exception {
-    HttpClient httpClient = Mockito.mock(HttpClient.class);
-    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
     Mockito.when(statusLine.getStatusCode()).thenReturn(500);
@@ -667,8 +667,8 @@ public void testRegisterBlobSnowflakeInternalErrorResponse() throws Exception {
             + "  } ]\n"
             + "}";
 
-    HttpClient httpClient = Mockito.mock(HttpClient.class);
-    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
     Mockito.when(statusLine.getStatusCode()).thenReturn(200);
@@ -724,8 +724,8 @@ public void testRegisterBlobSuccessResponse() throws Exception {
             + "  } ]\n"
             + "}";
 
-    HttpClient httpClient = Mockito.mock(HttpClient.class);
-    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
     Mockito.when(statusLine.getStatusCode()).thenReturn(200);
@@ -802,8 +802,8 @@ public void testRegisterBlobsRetries() throws Exception {
     String responseString = objectMapper.writeValueAsString(initialResponse);
     String retryResponseString = objectMapper.writeValueAsString(retryResponse);
 
-    HttpClient httpClient = Mockito.mock(HttpClient.class);
-    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
     Mockito.when(statusLine.getStatusCode()).thenReturn(200);
@@ -920,8 +920,8 @@ public void testRegisterBlobsRetriesSucceeds() throws Exception {
     String responseString = objectMapper.writeValueAsString(initialResponse);
     String retryResponseString = objectMapper.writeValueAsString(retryResponse);
 
-    HttpClient httpClient = Mockito.mock(HttpClient.class);
-    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
     Mockito.when(statusLine.getStatusCode()).thenReturn(200);
@@ -997,8 +997,8 @@ public void testRegisterBlobResponseWithInvalidChannel() throws Exception {
             channel2Name,
             channel2Sequencer);
 
-    HttpClient httpClient = Mockito.mock(HttpClient.class);
-    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
     Mockito.when(statusLine.getStatusCode()).thenReturn(200);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
index 6b40e3dc0..e1eda268f 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
@@ -38,10 +38,10 @@
 import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ParameterProvider;
-import org.apache.http.HttpResponse;
 import org.apache.http.StatusLine;
-import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.entity.BasicHttpEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -246,8 +246,8 @@ public void testPutRemoteGCS() throws Exception {
   @Test
   public void testRefreshSnowflakeMetadataRemote() throws Exception {
     RequestBuilder mockBuilder = Mockito.mock(RequestBuilder.class);
-    HttpClient mockClient = Mockito.mock(HttpClient.class);
-    HttpResponse mockResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient mockClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse mockResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine mockStatusLine = Mockito.mock(StatusLine.class);
     Mockito.when(mockStatusLine.getStatusCode()).thenReturn(200);
 
@@ -284,8 +284,8 @@ public void testRefreshSnowflakeMetadataRemote() throws Exception {
   @Test
   public void testFetchSignedURL() throws Exception {
     RequestBuilder mockBuilder = Mockito.mock(RequestBuilder.class);
-    HttpClient mockClient = Mockito.mock(HttpClient.class);
-    HttpResponse mockResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient mockClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse mockResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine mockStatusLine = Mockito.mock(StatusLine.class);
     Mockito.when(mockStatusLine.getStatusCode()).thenReturn(200);
 
@@ -322,8 +322,8 @@ public void testRefreshSnowflakeMetadataSynchronized() throws Exception {
             SnowflakeFileTransferAgent.getFileTransferMetadatas(exampleJson).get(0);
 
     RequestBuilder mockBuilder = Mockito.mock(RequestBuilder.class);
-    HttpClient mockClient = Mockito.mock(HttpClient.class);
-    HttpResponse mockResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient mockClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse mockResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine mockStatusLine = Mockito.mock(StatusLine.class);
     Mockito.when(mockStatusLine.getStatusCode()).thenReturn(200);
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
index 314235aa4..6f1653dc8 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
@@ -13,9 +13,9 @@
 import net.snowflake.ingest.TestUtils;
 import net.snowflake.ingest.connection.RequestBuilder;
 import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
 import org.apache.http.StatusLine;
-import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -33,8 +33,8 @@ public void testSuccessNoRetries() throws Exception {
     response.setChannels(new ArrayList<>());
     String responseString = objectMapper.writeValueAsString(response);
 
-    HttpClient httpClient = Mockito.mock(HttpClient.class);
-    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
     Mockito.when(statusLine.getStatusCode()).thenReturn(200);
@@ -74,8 +74,8 @@ public void testRetries() throws Exception {
     response.setChannels(new ArrayList<>());
     String responseString = objectMapper.writeValueAsString(response);
 
-    HttpClient httpClient = Mockito.mock(HttpClient.class);
-    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
     Mockito.when(statusLine.getStatusCode()).thenReturn(200);
@@ -128,8 +128,8 @@ public void testRetriesRecovery() throws Exception {
     successfulResponse.setChannels(new ArrayList<>());
     String successfulResponseString = objectMapper.writeValueAsString(successfulResponse);
 
-    HttpClient httpClient = Mockito.mock(HttpClient.class);
-    HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
     Mockito.when(statusLine.getStatusCode()).thenReturn(200);

From 146081782c2e122d6f5d098dc2fa044ff96b8195 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Sun, 24 Apr 2022 22:05:24 -0700
Subject: [PATCH 020/356] NO_SNOW: fix bug in channel.close  (#163)

1. Fix a bug in channel.close which may cause wrong result issue, the problem is that we remove the channel from the channel cache before calling flush, so it won't flush the buffers in this channel even if it has data in them
2. Add a small check to make sure row count shouldn't go beyond MAX INT value
3. Improve channel related error messages
4. Update streaming ingest IT tests to close the channel which will verify everything should be committed
---
 .../SnowflakeStreamingIngestChannel.java      |  2 +-
 .../SnowflakeStreamingIngestClient.java       |  8 ++-
 .../streaming/internal/ArrowRowBuffer.java    | 17 ++++++
 ...owflakeStreamingIngestChannelInternal.java |  8 +--
 .../net/snowflake/ingest/utils/Utils.java     |  2 +-
 .../ingest/ingest_error_messages.properties   |  4 +-
 .../streaming/internal/StreamingIngestIT.java | 55 +++++++++++++++++++
 7 files changed, 87 insertions(+), 9 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
index e1f0fdbc1..71358f405 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
@@ -67,7 +67,7 @@ public interface SnowflakeStreamingIngestChannel {
   boolean isClosed();
 
   /**
-   * Close the channel, this will make sure all the data in this channel is committed before closing
+   * Close the channel, this function will make sure all the data in this channel is committed
    *
    * @return a completable future which will be completed when the channel is closed
    */
diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
index 9dcc3b716..fec3cd147 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
@@ -27,6 +27,12 @@ public interface SnowflakeStreamingIngestClient extends AutoCloseable {
    */
   String getName();
 
-  /** @return a boolean to indicate whether the client is closed */
+  /**
+   * Check whether the client is closed or not, if you want to make sure all data are committed
+   * before closing, please call {@link SnowflakeStreamingIngestClient#close()} before closing the
+   * entire client
+   *
+   * @return a boolean to indicate whether the client is closed
+   */
   boolean isClosed();
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index ea70c50be..a3e15d02a 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -221,12 +221,23 @@ void setupSchema(List<ColumnMetadata> columns) {
    * synchronization
    */
   void close() {
+    long allocated = this.allocator.getAllocatedMemory();
     if (this.vectorsRoot != null) {
       this.vectorsRoot.close();
       this.tempVectorsRoot.close();
     }
     this.fields.clear();
     Utils.closeAllocator(this.allocator);
+
+    // If the channel is valid but still has leftover data, throw an exception because it should be
+    // cleaned up already before calling close
+    if (allocated > 0 && this.owningChannel.isValid()) {
+      throw new SFException(
+          ErrorCode.INTERNAL_ERROR,
+          String.format(
+              "Memory leaked=%d by allocator=%s, channel=%s",
+              allocated, this.allocator.toString(), this.owningChannel.getFullyQualifiedName()));
+    }
   }
 
   /** Reset the variables after each flush. Note that the caller needs to handle synchronization */
@@ -280,6 +291,9 @@ InsertValidationResponse insertRows(Iterable<Map<String, Object>> rows, String o
                     row, new SFException(e, ErrorCode.INTERNAL_ERROR, e.getMessage()), rowIndex));
           }
           rowIndex++;
+          if (this.rowCount == Integer.MAX_VALUE) {
+            throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
+          }
         }
       } else {
         // If the on_error option is ABORT, simply throw the first exception
@@ -302,6 +316,9 @@ InsertValidationResponse insertRows(Iterable<Map<String, Object>> rows, String o
           }
         }
         rowSize = tempRowSize;
+        if ((long) this.rowCount + tempRowCount >= Integer.MAX_VALUE) {
+          throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
+        }
         this.rowCount += tempRowCount;
         this.bufferSize += rowSize;
         this.statsMap.forEach(
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index bd90dfcd5..47b15d878 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -281,7 +281,7 @@ CompletableFuture<Void> flush(boolean closing) {
     // Skip this check for closing because we need to set the channel to closed first and then flush
     // in case there is any leftover rows
     if (isClosed() && !closing) {
-      throw new SFException(ErrorCode.CLOSED_CHANNEL);
+      throw new SFException(ErrorCode.CLOSED_CHANNEL, getFullyQualifiedName());
     }
 
     // Simply return if there is no data in the channel, this might not work if we support public
@@ -307,7 +307,6 @@ public CompletableFuture<Void> close() {
     }
 
     markClosed();
-    this.owningClient.removeChannelIfSequencersMatch(this);
     return flush(true)
         .thenRunAsync(
             () -> {
@@ -316,6 +315,7 @@ public CompletableFuture<Void> close() {
                       Collections.singletonList(this));
 
               this.arrowBuffer.close();
+              this.owningClient.removeChannelIfSequencersMatch(this);
 
               // Throw an exception if the channel has any uncommitted rows
               if (!uncommittedChannels.isEmpty()) {
@@ -388,7 +388,7 @@ public InsertValidationResponse insertRows(
     checkValidation();
 
     if (isClosed()) {
-      throw new SFException(ErrorCode.CLOSED_CHANNEL);
+      throw new SFException(ErrorCode.CLOSED_CHANNEL, getFullyQualifiedName());
     }
 
     InsertValidationResponse response = this.arrowBuffer.insertRows(rows, offsetToken);
@@ -470,7 +470,7 @@ private void checkValidation() {
     if (!isValid()) {
       this.owningClient.removeChannelIfSequencersMatch(this);
       this.arrowBuffer.close();
-      throw new SFException(ErrorCode.INVALID_CHANNEL);
+      throw new SFException(ErrorCode.INVALID_CHANNEL, getFullyQualifiedName());
     }
   }
 
diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java
index 31a6f021e..878668c30 100644
--- a/src/main/java/net/snowflake/ingest/utils/Utils.java
+++ b/src/main/java/net/snowflake/ingest/utils/Utils.java
@@ -266,7 +266,7 @@ public static byte[] toByteArray(int value) {
     };
   }
 
-  /** Utility function to check whether a streing is null or empty */
+  /** Utility function to check whether a string is null or empty */
   public static boolean isNullOrEmpty(String string) {
     return string == null || string.isEmpty();
   }
diff --git a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
index edf73a09c..550b3355a 100644
--- a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
+++ b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
@@ -15,8 +15,8 @@
 0010=Missing {0} in config file.
 0011=Failed to upload blob.
 0012=Failed to cleanup resources during {0}.
-0013=Channel is invalid and might contain uncommitted rows, please consider reopening the channel to restart.
-0014=Channel is closed, please reopen the channel to restart.
+0013=Channel {0} is invalid and might contain uncommitted rows, please consider reopening the channel to restart.
+0014=Channel {0} is closed, please reopen the channel to restart.
 0015=Invalid Snowflake URL, URL format: 'https://<account_name>.<region_name>.snowflakecomputing.com:443', 'https://' and ':443' are optional.
 0016=Client is closed, please recreate to restart.
 0017=Invalid private key, private key should be a valid PEM RSA private key.
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 009127827..f50e12c9b 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -108,6 +108,9 @@ public void testSimpleIngest() throws Exception {
       verifyInsertValidationResponse(channel1.insertRow(row, Integer.toString(val)));
     }
 
+    // Close the channel after insertion
+    channel1.close().get();
+
     for (int i = 1; i < 15; i++) {
       if (channel1.getLatestCommittedOffsetToken() != null
           && channel1.getLatestCommittedOffsetToken().equals("999")) {
@@ -183,6 +186,9 @@ public void testParameterOverrides() throws Exception {
       verifyInsertValidationResponse(channel1.insertRow(row, Integer.toString(val)));
     }
 
+    // Close the channel after insertion
+    channel1.close().get();
+
     for (int i = 1; i < 15; i++) {
       if (channel1.getLatestCommittedOffsetToken() != null
           && channel1.getLatestCommittedOffsetToken().equals("9")) {
@@ -249,6 +255,9 @@ public void testCollation() throws Exception {
     row.put("noncol", "a");
     verifyInsertValidationResponse(channel1.insertRow(row, "2"));
 
+    // Close the channel after insertion
+    channel1.close().get();
+
     for (int i = 1; i < 15; i++) {
       if (channel1.getLatestCommittedOffsetToken() != null
           && channel1.getLatestCommittedOffsetToken().equals("2")) {
@@ -296,6 +305,9 @@ public void testDecimalColumnIngest() throws Exception {
     row.put("tinyfloat", BigInteger.valueOf(10).pow(35));
     verifyInsertValidationResponse(channel1.insertRow(row, "1"));
 
+    // Close the channel after insertion
+    channel1.close().get();
+
     for (int i = 1; i < 15; i++) {
       if (channel1.getLatestCommittedOffsetToken() != null
           && channel1.getLatestCommittedOffsetToken().equals("1")) {
@@ -372,6 +384,9 @@ public void testTimeColumnIngest() throws Exception {
     row.put("tntzbig", "2031-01-01 09:00:00.12345678");
     verifyInsertValidationResponse(channel1.insertRow(row, "1"));
 
+    // Close the channel after insertion
+    channel1.close().get();
+
     for (int i = 1; i < 15; i++) {
       if (channel1.getLatestCommittedOffsetToken() != null
           && channel1.getLatestCommittedOffsetToken().equals("1")) {
@@ -459,6 +474,9 @@ public void testMultiColumnIngest() throws Exception {
     row.put("d", "1967-06-23 01:01:01");
     verifyInsertValidationResponse(channel1.insertRow(row, "1"));
 
+    // Close the channel after insertion
+    channel1.close().get();
+
     for (int i = 1; i < 15; i++) {
       if (channel1.getLatestCommittedOffsetToken() != null
           && channel1.getLatestCommittedOffsetToken().equals("1")) {
@@ -531,6 +549,10 @@ public void testNullableColumns() throws Exception {
 
     verifyInsertValidationResponse(channel2.insertRow(row3, "1"));
 
+    // Close the channel after insertion
+    channel1.close().get();
+    channel2.close().get();
+
     for (int i = 1; i < 15; i++) {
       if (channel1.getLatestCommittedOffsetToken() != null
           && channel1.getLatestCommittedOffsetToken().equals("3")
@@ -605,6 +627,11 @@ public void testMultiThread() throws Exception {
     CompletableFuture joined = CompletableFuture.allOf(futures);
     joined.get();
 
+    // Close the channel after insertion
+    for (SnowflakeStreamingIngestChannel channel : channelList) {
+      channel.close().get();
+    }
+
     for (int i = 1; i < 15; i++) {
       if (channelList.stream()
           .allMatch(
@@ -704,6 +731,9 @@ public void testTwoClientsOneChannel() throws Exception {
     row.put("c1", "8");
     verifyInsertValidationResponse(channelA2.insertRow(row, "8"));
 
+    // Close the channel after insertion
+    channelA2.close().get();
+
     for (int i = 1; i < 15; i++) {
       if (channelA2.getLatestCommittedOffsetToken() != null
           && channelA2.getLatestCommittedOffsetToken().equals("8")) {
@@ -777,6 +807,9 @@ public void testAbortOnErrorOption() throws Exception {
     row7.put("c1", 7);
     channel.insertRow(row7, "7");
 
+    // Close the channel after insertion
+    channel.close().get();
+
     for (int i = 1; i < 15; i++) {
       if (channel.getLatestCommittedOffsetToken() != null
           && channel.getLatestCommittedOffsetToken().equals("7")) {
@@ -798,6 +831,28 @@ public void testAbortOnErrorOption() throws Exception {
     Assert.fail("Row sequencer not updated before timeout");
   }
 
+  @Test
+  public void testChannelClose() throws Exception {
+    OpenChannelRequest request1 =
+        OpenChannelRequest.builder("CHANNEL")
+            .setDBName(TEST_DB)
+            .setSchemaName(TEST_SCHEMA)
+            .setTableName(TEST_TABLE)
+            .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+            .build();
+
+    // Open a streaming ingest channel from the given client
+    SnowflakeStreamingIngestChannel channel1 = client.openChannel(request1);
+    for (int val = 0; val < 1000; val++) {
+      Map<String, Object> row = new HashMap<>();
+      row.put("c1", Integer.toString(val));
+      verifyInsertValidationResponse(channel1.insertRow(row, Integer.toString(val)));
+    }
+
+    // Close the channel to make sure everything is committed
+    channel1.close().get();
+  }
+
   /** Verify the insert validation response and throw the exception if needed */
   private void verifyInsertValidationResponse(InsertValidationResponse response) {
     if (response.hasErrors()) {

From 8b901bfe5072e46521266f362aee11d6b0142744 Mon Sep 17 00:00:00 2001
From: sfc-gh-jfan <jiazhen.fan@snowflake.com>
Date: Mon, 25 Apr 2022 19:45:22 -0700
Subject: [PATCH 021/356] PRODSEC-1251 Whitesource Transition to Snyk

---
 .../{whitesource-issue.yml => snyk-issue.yml} | 13 ++++++-------
 .../{whitesource.yml => snyk-pr.yml}          | 10 +++++-----
 .whitesource                                  | 19 -------------------
 3 files changed, 11 insertions(+), 31 deletions(-)
 rename .github/workflows/{whitesource-issue.yml => snyk-issue.yml} (54%)
 rename .github/workflows/{whitesource.yml => snyk-pr.yml} (71%)
 delete mode 100644 .whitesource

diff --git a/.github/workflows/whitesource-issue.yml b/.github/workflows/snyk-issue.yml
similarity index 54%
rename from .github/workflows/whitesource-issue.yml
rename to .github/workflows/snyk-issue.yml
index 812a1c741..b54275bfc 100644
--- a/.github/workflows/whitesource-issue.yml
+++ b/.github/workflows/snyk-issue.yml
@@ -1,16 +1,15 @@
-
-name: whitesource-issue
+name: Snyk Issue
 
 on:
   issues:
     types: [opened, reopened]
 
-concurrency: whitesource-issue
+concurrency: snyk-issue
 
 jobs:
   whitesource:
     runs-on: ubuntu-latest
-    if: ${{ github.event.issue.user.login == 'whitesource-for-github-com[bot]' }}
+    if: ${{ github.event.issue.user.login == 'sfc-gh-snyk-sca-sa' }}
     steps:
     - name: checkout action
       uses: actions/checkout@v2
@@ -20,7 +19,7 @@ jobs:
         path: whitesource-actions
 
     - name: Jira Creation
-      uses: ./whitesource-actions/whitesource-issue
+      uses: ./whitesource-actions/snyk-issue
       with:
-        jira_token: ${{ secrets.JIRA_TOKEN_PUBLIC_REPO }}
-        gh_token: ${{ secrets.GITHUB_TOKEN }}
+        jira_token: ${{ secrets.JIRA_TOKEN }}
+        gh_token: ${{ secrets.GITHUB_TOKEN }}
\ No newline at end of file
diff --git a/.github/workflows/whitesource.yml b/.github/workflows/snyk-pr.yml
similarity index 71%
rename from .github/workflows/whitesource.yml
rename to .github/workflows/snyk-pr.yml
index 860b5f850..0fb62a819 100644
--- a/.github/workflows/whitesource.yml
+++ b/.github/workflows/snyk-pr.yml
@@ -1,4 +1,4 @@
-name: whitesource
+name: snyk-pr
 on:
   pull_request:
     branches:
@@ -6,7 +6,7 @@ on:
 jobs:
   whitesource:
     runs-on: ubuntu-latest
-    if: ${{ github.event.pull_request.user.login == 'whitesource-for-github-com[bot]' }}
+    if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }}
     steps:
     - name: checkout
       uses: actions/checkout@v2
@@ -22,8 +22,8 @@ jobs:
         path: whitesource-actions
 
     - name: PR
-      uses: ./whitesource-actions/whitesource-pr
+      uses: ./whitesource-actions/snyk-pr
       with:
-        jira_token: ${{ secrets.JIRA_TOKEN_PUBLIC_REPO }}
+        jira_token: ${{ secrets.JIRA_TOKEN }}
         gh_token: ${{ secrets.GITHUB_TOKEN }}
-        amend: false # true if you want the commit to be amended with the JIRA number
+        amend: false # true if you want the commit to be amended with the JIRA number
\ No newline at end of file
diff --git a/.whitesource b/.whitesource
deleted file mode 100644
index 6b7f0e839..000000000
--- a/.whitesource
+++ /dev/null
@@ -1,19 +0,0 @@
-{
-  "scanSettings": {
-    "configMode": "AUTO",
-    "configExternalURL": "",
-    "projectToken": "",
-    "baseBranches": []
-  },
-  "checkRunSettings": {
-    "vulnerableCheckRunConclusionLevel": "failure",
-    "displayMode": "diff"
-  },
-  "issueSettings": {
-    "minSeverityLevel": "LOW"
-  },
-  "remediateSettings": {
-    "enableRenovate": false,
-    "commitMessagePrefix": "SNOW-470176"
-  }
-}
\ No newline at end of file

From 6d92fe542f600933beebd278fa49e35cc018987e Mon Sep 17 00:00:00 2001
From: sfc-gh-jfan <jiazhen.fan@snowflake.com>
Date: Wed, 27 Apr 2022 16:09:12 -0700
Subject: [PATCH 022/356] minor update

---
 .github/workflows/snyk-issue.yml | 14 +++++++++-----
 .github/workflows/snyk-pr.yml    |  4 +++-
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/snyk-issue.yml b/.github/workflows/snyk-issue.yml
index b54275bfc..9641ba37b 100644
--- a/.github/workflows/snyk-issue.yml
+++ b/.github/workflows/snyk-issue.yml
@@ -1,15 +1,14 @@
 name: Snyk Issue
 
 on:
-  issues:
-    types: [opened, reopened]
+  schedule:
+    - cron: '* */12 * * *'
 
 concurrency: snyk-issue
 
 jobs:
   whitesource:
     runs-on: ubuntu-latest
-    if: ${{ github.event.issue.user.login == 'sfc-gh-snyk-sca-sa' }}
     steps:
     - name: checkout action
       uses: actions/checkout@v2
@@ -17,9 +16,14 @@ jobs:
         repository: snowflakedb/whitesource-actions
         token: ${{ secrets.WHITESOURCE_ACTION_TOKEN }}
         path: whitesource-actions
-
+    - name: set-env
+      run: echo "REPO=$(basename $GITHUB_REPOSITORY)" >> $GITHUB_ENV
     - name: Jira Creation
       uses: ./whitesource-actions/snyk-issue
       with:
+        snyk_org: 91a1c3d0-0729-4a98-8210-28087460f1e3
+        snyk_token:  ${{ secrets.SNYK_GITHUB_INTEGRATION_TOKEN }}
         jira_token: ${{ secrets.JIRA_TOKEN }}
-        gh_token: ${{ secrets.GITHUB_TOKEN }}
\ No newline at end of file
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
diff --git a/.github/workflows/snyk-pr.yml b/.github/workflows/snyk-pr.yml
index 0fb62a819..02b5f173d 100644
--- a/.github/workflows/snyk-pr.yml
+++ b/.github/workflows/snyk-pr.yml
@@ -23,7 +23,9 @@ jobs:
 
     - name: PR
       uses: ./whitesource-actions/snyk-pr
+      env:
+        PR_TITLE: ${{ github.event.pull_request.title }}
       with:
         jira_token: ${{ secrets.JIRA_TOKEN }}
         gh_token: ${{ secrets.GITHUB_TOKEN }}
-        amend: false # true if you want the commit to be amended with the JIRA number
\ No newline at end of file
+        amend: false # true if you want the commit to be amended with the JIRA number

From 34ae2bb5210c28195b668cc9d3d4f4ac7804fa93 Mon Sep 17 00:00:00 2001
From: sfc-gh-jfan <jiazhen.fan@snowflake.com>
Date: Wed, 4 May 2022 14:54:31 -0700
Subject: [PATCH 023/356] minor update

---
 .github/workflows/snyk-issue.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/snyk-issue.yml b/.github/workflows/snyk-issue.yml
index 9641ba37b..d2ef124d7 100644
--- a/.github/workflows/snyk-issue.yml
+++ b/.github/workflows/snyk-issue.yml
@@ -21,7 +21,7 @@ jobs:
     - name: Jira Creation
       uses: ./whitesource-actions/snyk-issue
       with:
-        snyk_org: 91a1c3d0-0729-4a98-8210-28087460f1e3
+        snyk_org: ${{ secrets.SNYK_ORG_ID }}
         snyk_token:  ${{ secrets.SNYK_GITHUB_INTEGRATION_TOKEN }}
         jira_token: ${{ secrets.JIRA_TOKEN }}
       env:

From e70e98abd51d0916b240586bc2f586a76fa35fb0 Mon Sep 17 00:00:00 2001
From: sfc-gh-jfan <jiazhen.fan@snowflake.com>
Date: Wed, 15 Jun 2022 12:53:30 -0700
Subject: [PATCH 024/356] update snyk org id

---
 .github/workflows/snyk-issue.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/snyk-issue.yml b/.github/workflows/snyk-issue.yml
index d2ef124d7..9a45b0c93 100644
--- a/.github/workflows/snyk-issue.yml
+++ b/.github/workflows/snyk-issue.yml
@@ -21,7 +21,7 @@ jobs:
     - name: Jira Creation
       uses: ./whitesource-actions/snyk-issue
       with:
-        snyk_org: ${{ secrets.SNYK_ORG_ID }}
+        snyk_org: ${{ secrets.SNYK_ORG_ID_PUBLIC_REPO }}
         snyk_token:  ${{ secrets.SNYK_GITHUB_INTEGRATION_TOKEN }}
         jira_token: ${{ secrets.JIRA_TOKEN }}
       env:

From 41ddc545a3cf5ad61613d9206e295f57e6217aa6 Mon Sep 17 00:00:00 2001
From: sfc-gh-jfan <jiazhen.fan@snowflake.com>
Date: Wed, 15 Jun 2022 12:56:07 -0700
Subject: [PATCH 025/356] checkout v3

---
 .github/workflows/snyk-issue.yml | 2 +-
 .github/workflows/snyk-pr.yml    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/snyk-issue.yml b/.github/workflows/snyk-issue.yml
index 9a45b0c93..7131bd14b 100644
--- a/.github/workflows/snyk-issue.yml
+++ b/.github/workflows/snyk-issue.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: checkout action
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         repository: snowflakedb/whitesource-actions
         token: ${{ secrets.WHITESOURCE_ACTION_TOKEN }}
diff --git a/.github/workflows/snyk-pr.yml b/.github/workflows/snyk-pr.yml
index 02b5f173d..9a3c117f1 100644
--- a/.github/workflows/snyk-pr.yml
+++ b/.github/workflows/snyk-pr.yml
@@ -9,13 +9,13 @@ jobs:
     if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }}
     steps:
     - name: checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         ref: ${{ github.event.pull_request.head.ref }}
         fetch-depth: 0
 
     - name: checkout action
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         repository: snowflakedb/whitesource-actions
         token: ${{ secrets.WHITESOURCE_ACTION_TOKEN }}

From 9eb7e92e821c940327f4b112095599b981337819 Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Tue, 26 Apr 2022 14:22:40 +0000
Subject: [PATCH 026/356] Fix logging project properties load exception

---
 .../ingest/connection/RequestBuilder.java     | 36 +++++--------------
 1 file changed, 9 insertions(+), 27 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 034cebeb6..3c05e37cd 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -10,14 +10,9 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.FileNotFoundException;
-import java.io.IOException;
 import java.io.InputStream;
-import java.io.UncheckedIOException;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Paths;
 import java.security.KeyPair;
 import java.util.List;
 import java.util.Map;
@@ -25,10 +20,7 @@
 import java.util.Properties;
 import java.util.UUID;
 import net.snowflake.ingest.SimpleIngestManager;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.SnowflakeURL;
-import net.snowflake.ingest.utils.StagedFileWrapper;
+import net.snowflake.ingest.utils.*;
 import org.apache.http.HttpHeaders;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
@@ -257,28 +249,18 @@ private static Properties loadProperties() {
     Properties properties = new Properties();
     properties.put("version", DEFAULT_VERSION);
 
-    try {
-      URL res = SimpleIngestManager.class.getClassLoader().getResource(RESOURCES_FILE);
-      if (res == null) {
-        throw new UncheckedIOException(new FileNotFoundException(RESOURCES_FILE));
-      }
-
-      URI uri;
-      try {
-        uri = res.toURI();
-      } catch (URISyntaxException ex) {
-        throw new IllegalArgumentException(ex);
-      }
-
-      try (InputStream is = Files.newInputStream(Paths.get(uri))) {
-        properties.load(is);
-      } catch (IOException ex) {
-        throw new UncheckedIOException("Failed to load resource", ex);
+    try (InputStream is =
+        SimpleIngestManager.class.getClassLoader().getResourceAsStream(RESOURCES_FILE)) {
+      if (is == null) {
+        throw new FileNotFoundException(RESOURCES_FILE);
       }
+      properties.load(is);
     } catch (Exception e) {
-      LOGGER.warn("Could not read version info: " + e.toString());
+      LOGGER.warn("Could not read version info, use default version " + DEFAULT_VERSION, e);
+      return properties;
     }
 
+    LOGGER.debug("Loaded project version " + properties.getProperty("version"));
     return properties;
   }
 

From d73446746ebc2461d3c236501ae3ac60d0a46ff3 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Tue, 26 Apr 2022 14:38:08 -0700
Subject: [PATCH 027/356] NO-SNOW: increase retry count due to server side
 Commit Service not making progress  (#172)

As part of the channel.close API, we will keep checking whether the server side is making progress (committing offset), and throw an error with certain timeout if no progress. The current timeout is set to 5s, and according to our recent resilience testing, somehow the Commit Service does not able to make any progress. While the team is investigating the root cause, increase the retry timeout to 10s (retry interval * retry count) instead.
---
 src/main/java/net/snowflake/ingest/utils/Constants.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index d7edf88b8..437f7f463 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -44,7 +44,7 @@ public class Constants {
   public static final int MAX_THREAD_COUNT = Integer.MAX_VALUE;
   public static final int CPU_IO_TIME_RATIO = 1;
   public static final String CLIENT_CONFIGURE_ENDPOINT = "/v1/streaming/client/configure/";
-  public static final int COMMIT_MAX_RETRY_COUNT = 10;
+  public static final int COMMIT_MAX_RETRY_COUNT = 20;
   public static final int COMMIT_RETRY_INTERVAL_IN_MS = 500;
   public static final int RESPONSE_ROW_SEQUENCER_IS_COMMITTED =
       26; // Don't change, should match server side

From 431ab71472fd4f35bf737eccfcc19b9c37829afb Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 27 Apr 2022 15:20:38 -0700
Subject: [PATCH 028/356] 1.0.2-beta.1 release (#169)

Release note:

- On top of 1.0.2-beta
- Fix a critical bug in channel.close to avoid wrong result
- Retry HTTP APIs on certain status code returned from SF
---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 04a5d74e5..18d831eec 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
     <!-- Arifact name and version information -->
     <groupId>net.snowflake</groupId>
     <artifactId>snowflake-ingest-sdk</artifactId>
-    <version>1.0.2-beta</version>
+    <version>1.0.2-beta.1</version>
     <packaging>jar</packaging>
     <name>Snowflake Ingest SDK</name>
     <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 3c05e37cd..c71312db3 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -119,7 +119,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.0.2-beta";
+  public static final String DEFAULT_VERSION = "1.0.2-beta.1";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 190385ec855bb780754af9b5cbea9a96740add48 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Mon, 2 May 2022 13:39:19 -0700
Subject: [PATCH 029/356] NO-SNOW: accept account names with hyphen (#174)

Currently, we will error out when a valid account name contains a hyphen (example: pm-connectors.snowflakecomputing.com), this change fixes this issue and the regex logic is from KC
---
 .../net/snowflake/ingest/utils/SnowflakeURL.java     |  2 +-
 .../ingest/streaming/internal/SnowflakeURLTest.java  | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/main/java/net/snowflake/ingest/utils/SnowflakeURL.java b/src/main/java/net/snowflake/ingest/utils/SnowflakeURL.java
index 533b6db8f..cfcc73056 100644
--- a/src/main/java/net/snowflake/ingest/utils/SnowflakeURL.java
+++ b/src/main/java/net/snowflake/ingest/utils/SnowflakeURL.java
@@ -64,7 +64,7 @@ private SnowflakeURL(SnowflakeURLBuilder builder) {
    */
   public SnowflakeURL(String urlStr) {
     Pattern pattern =
-        Pattern.compile("^(https?://)?((([\\w\\d]+)(\\" + ".[\\w\\d-]+){2,})(:(\\d+))?)/?$");
+        Pattern.compile("^(https?://)?((([\\w\\d-]+)(\\.[\\w\\d-]+){2,})(:(\\d+))?)/?$");
 
     Matcher matcher = pattern.matcher(urlStr.trim().toLowerCase());
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeURLTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeURLTest.java
index 837501083..de1942669 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeURLTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeURLTest.java
@@ -37,6 +37,18 @@ public void testParseSane() throws Exception {
     Assert.assertEquals(8082, url.getPort());
     Assert.assertEquals("http", url.getScheme());
     Assert.assertEquals("snowflake.dev.local:8082", url.getFullUrl());
+
+    url = new SnowflakeURL("https://pm-connectors.snowflakecomputing.com:443");
+    Assert.assertEquals(443, url.getPort());
+    Assert.assertEquals("https", url.getScheme());
+    Assert.assertEquals("pm-connectors.snowflakecomputing.com:443", url.getFullUrl());
+    Assert.assertEquals("pm-connectors", url.getAccount());
+
+    url = new SnowflakeURL("https://pm_connectors.snowflakecomputing.com:443");
+    Assert.assertEquals(443, url.getPort());
+    Assert.assertEquals("https", url.getScheme());
+    Assert.assertEquals("pm_connectors.snowflakecomputing.com:443", url.getFullUrl());
+    Assert.assertEquals("pm_connectors", url.getAccount());
   }
 
   @Test

From a5c4d7f39d63846d9ddcb8f9e0028b3c0cfa3f54 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Fri, 6 May 2022 11:14:22 -0700
Subject: [PATCH 030/356] NO-SNOW: A few small fixes on Snowpipe Streaming PrPr
 (#175)

1. Add a profile example file (profile_streaming.json.example) specific for Snowpipe Streaming, since there was customer confusions that NOT everything from profile.json.example makes sense for Snowpipe Streaming
2. Move the properties file verification logic all inside createProperties
3. Remove the usage of getPropertiesFromJson from the example since it won't work outside of the SDK due to the jackson dependency is shaded
---
 profile.json.example                          |  8 +--
 profile_streaming.json.example                |  9 +++
 ...SnowflakeStreamingIngestClientFactory.java | 30 +-------
 .../SnowflakeStreamingIngestExample.java      | 16 +++--
 .../net/snowflake/ingest/utils/Utils.java     | 68 +++++++------------
 .../java/net/snowflake/ingest/TestUtils.java  |  8 +++
 .../SnowflakeStreamingIngestChannelTest.java  |  6 +-
 .../SnowflakeStreamingIngestClientTest.java   |  5 +-
 8 files changed, 67 insertions(+), 83 deletions(-)
 create mode 100644 profile_streaming.json.example

diff --git a/profile.json.example b/profile.json.example
index 9f99ad0c1..4cfb728ba 100644
--- a/profile.json.example
+++ b/profile.json.example
@@ -1,15 +1,15 @@
 {
   "user": "user name",
-  "url": "https://account_name.snowflakecomputing.com:443"
+  "url": "https://account_name.snowflakecomputing.com:443",
   "account": "account_name",
   "private_key": "PEM Private Key",
   "port": 443,
   "host": "account_name.snowflakecomputing.com",
   "schema": "schema",
   "scheme": "https",
-  "database": "database name",
+  "database": "database_name",
   "connect_string": "jdbc:snowflake://account_name.snowflakecomputing.com:443",
   "ssl": "on",
-  "warehouse": "warehouse name"
-  "role": "accountadmin"
+  "warehouse": "warehouse_name"
+  "role": "role_name"
 }
\ No newline at end of file
diff --git a/profile_streaming.json.example b/profile_streaming.json.example
new file mode 100644
index 000000000..a94994b4b
--- /dev/null
+++ b/profile_streaming.json.example
@@ -0,0 +1,9 @@
+{
+  "user": "user name",
+  "url": "https://account_name.snowflakecomputing.com:443", // This is required, or it can be constructed from host, scheme and port
+  "private_key": "PEM Private Key",
+  "port": 443,
+  "host": "account_name.snowflakecomputing.com",
+  "scheme": "https",
+  "role": "role_name"
+}
\ No newline at end of file
diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java
index d99f54e56..9d06ab86e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java
@@ -8,8 +8,6 @@
 import java.util.Properties;
 import net.snowflake.ingest.streaming.internal.SnowflakeStreamingIngestClientInternal;
 import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.SnowflakeURL;
 import net.snowflake.ingest.utils.Utils;
 
@@ -48,33 +46,11 @@ public SnowflakeStreamingIngestClient build() {
       Utils.assertStringNotNullOrEmpty("client name", this.name);
       Utils.assertNotNull("connection properties", this.prop);
 
-      if (!this.prop.containsKey(Constants.ACCOUNT_URL)) {
-        if (!this.prop.containsKey(Constants.HOST)) {
-          throw new SFException(ErrorCode.MISSING_CONFIG, "host");
-        }
-        if (!this.prop.containsKey(Constants.SCHEME)) {
-          throw new SFException(ErrorCode.MISSING_CONFIG, "scheme");
-        }
-        if (!this.prop.containsKey(Constants.PORT)) {
-          throw new SFException(ErrorCode.MISSING_CONFIG, "port");
-        }
+      Properties prop = Utils.createProperties(this.prop);
+      SnowflakeURL accountURL = new SnowflakeURL(prop.getProperty(Constants.ACCOUNT_URL));
 
-        this.prop.put(
-            Constants.ACCOUNT_URL,
-            Utils.constructAccountUrl(
-                this.prop.get(Constants.SCHEME).toString(),
-                this.prop.get(Constants.HOST).toString(),
-                Integer.parseInt(prop.get(Constants.PORT).toString())));
-      }
-
-      if (!this.prop.containsKey(Constants.ROLE)) {
-        throw new SFException(ErrorCode.MISSING_CONFIG, "role");
-      }
-
-      SnowflakeURL accountURL = new SnowflakeURL(this.prop.getProperty(Constants.ACCOUNT_URL));
-      Properties prop = Utils.createProperties(this.prop, accountURL.sslEnabled());
       return new SnowflakeStreamingIngestClientInternal(
-          this.name, accountURL, prop, parameterOverrides);
+          this.name, accountURL, prop, this.parameterOverrides);
     }
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
index 85409ce20..1ce845d01 100644
--- a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
+++ b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
@@ -4,11 +4,12 @@
 
 package net.snowflake.ingest.streaming.example;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Properties;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
@@ -16,7 +17,6 @@
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
-import net.snowflake.ingest.utils.Utils;
 
 /** Examples on how to use the Streaming Ingest client APIs */
 public class SnowflakeStreamingIngestExample {
@@ -24,13 +24,17 @@ public class SnowflakeStreamingIngestExample {
   private static final ObjectMapper mapper = new ObjectMapper();
 
   public static void main(String[] args) throws Exception {
-    ObjectNode profile =
-        (ObjectNode) mapper.readTree(new String(Files.readAllBytes(Paths.get(PROFILE_PATH))));
-    Properties prop = Utils.getPropertiesFromJson(profile);
+    Properties props = new Properties();
+    Iterator<Map.Entry<String, JsonNode>> propIt =
+        mapper.readTree(new String(Files.readAllBytes(Paths.get(PROFILE_PATH)))).fields();
+    while (propIt.hasNext()) {
+      Map.Entry<String, JsonNode> prop = propIt.next();
+      props.put(prop.getKey(), prop.getValue().asText());
+    }
 
     // Create a streaming ingest client
     try (SnowflakeStreamingIngestClient client =
-        SnowflakeStreamingIngestClientFactory.builder("CLIENT").setProperties(prop).build()) {
+        SnowflakeStreamingIngestClientFactory.builder("CLIENT").setProperties(props).build()) {
       // Create an open channel request on table T_STREAMINGINGEST
       OpenChannelRequest request1 =
           OpenChannelRequest.builder("CHANNEL")
diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java
index 878668c30..5b421f9d1 100644
--- a/src/main/java/net/snowflake/ingest/utils/Utils.java
+++ b/src/main/java/net/snowflake/ingest/utils/Utils.java
@@ -5,12 +5,9 @@
 package net.snowflake.ingest.utils;
 
 import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
-import static net.snowflake.ingest.utils.Constants.ROLE;
-import static net.snowflake.ingest.utils.Constants.SSL;
 import static net.snowflake.ingest.utils.Constants.USER;
 
 import com.codahale.metrics.Timer;
-import com.fasterxml.jackson.databind.node.ObjectNode;
 import java.io.StringReader;
 import java.security.KeyFactory;
 import java.security.KeyPair;
@@ -23,7 +20,6 @@
 import java.security.spec.PKCS8EncodedKeySpec;
 import java.security.spec.RSAPublicKeySpec;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Properties;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
@@ -70,10 +66,9 @@ public static void assertNotNull(String name, Object value) throws SFException {
    * Create a Properties for snowflake connection
    *
    * @param inputProp input property map
-   * @param sslEnabled if ssl is enabled
    * @return a Properties instance
    */
-  public static Properties createProperties(Properties inputProp, boolean sslEnabled) {
+  public static Properties createProperties(Properties inputProp) {
     Properties properties = new Properties();
 
     // decrypt rsa key
@@ -91,14 +86,8 @@ public static Properties createProperties(Properties inputProp, boolean sslEnabl
         case Constants.PRIVATE_KEY_PASSPHRASE:
           privateKeyPassphrase = val;
           break;
-        case Constants.USER:
-          properties.put(USER, val);
-          break;
-        case Constants.ROLE:
-          properties.put(ROLE, val);
-          break;
         default:
-          properties.put(key, val);
+          properties.put(key.toLowerCase(), val);
       }
     }
 
@@ -108,13 +97,6 @@ public static Properties createProperties(Properties inputProp, boolean sslEnabl
       properties.put(JDBC_PRIVATE_KEY, parsePrivateKey(privateKey));
     }
 
-    // set ssl
-    if (sslEnabled) {
-      properties.put(SSL, "on");
-    } else {
-      properties.put(SSL, "off");
-    }
-
     if (!properties.containsKey(JDBC_PRIVATE_KEY)) {
       throw new SFException(ErrorCode.MISSING_CONFIG, "private_key");
     }
@@ -123,6 +105,29 @@ public static Properties createProperties(Properties inputProp, boolean sslEnabl
       throw new SFException(ErrorCode.MISSING_CONFIG, "user");
     }
 
+    if (!properties.containsKey(Constants.ACCOUNT_URL)) {
+      if (!properties.containsKey(Constants.HOST)) {
+        throw new SFException(ErrorCode.MISSING_CONFIG, "host");
+      }
+      if (!properties.containsKey(Constants.SCHEME)) {
+        throw new SFException(ErrorCode.MISSING_CONFIG, "scheme");
+      }
+      if (!properties.containsKey(Constants.PORT)) {
+        throw new SFException(ErrorCode.MISSING_CONFIG, "port");
+      }
+
+      properties.put(
+          Constants.ACCOUNT_URL,
+          Utils.constructAccountUrl(
+              properties.get(Constants.SCHEME).toString(),
+              properties.get(Constants.HOST).toString(),
+              Integer.parseInt(properties.get(Constants.PORT).toString())));
+    }
+
+    if (!properties.containsKey(Constants.ROLE)) {
+      throw new SFException(ErrorCode.MISSING_CONFIG, "role");
+    }
+
     return properties;
   }
 
@@ -131,29 +136,6 @@ public static String constructAccountUrl(String scheme, String host, int port) {
     return String.format("%s://%s:%d", scheme, host, port);
   }
 
-  /** Get the properties out from a json node */
-  public static Properties getPropertiesFromJson(ObjectNode json) {
-    Properties props = new Properties();
-    Optional.ofNullable(json.get(Constants.USER))
-        .ifPresent(u -> props.put(Constants.USER, u.asText()));
-    Optional.ofNullable(json.get(Constants.PRIVATE_KEY))
-        .ifPresent(u -> props.put(Constants.PRIVATE_KEY, u.asText()));
-    Optional.ofNullable(json.get(Constants.ROLE)).ifPresent(u -> props.put(ROLE, u.asText()));
-    Optional.ofNullable(json.get(Constants.PRIVATE_KEY_PASSPHRASE))
-        .ifPresent(u -> props.put(Constants.PRIVATE_KEY_PASSPHRASE, u.asText()));
-
-    Optional.ofNullable(json.get(Constants.SCHEME))
-        .ifPresent(u -> props.put(Constants.SCHEME, u.asText()));
-    Optional.ofNullable(json.get(Constants.HOST))
-        .ifPresent(u -> props.put(Constants.HOST, u.asText()));
-    Optional.ofNullable(json.get(Constants.PORT))
-        .ifPresent(u -> props.put(Constants.PORT, u.asInt()));
-    Optional.ofNullable(json.get(Constants.ACCOUNT_URL))
-        .ifPresent(u -> props.put(Constants.ACCOUNT_URL, u.asText()));
-
-    return props;
-  }
-
   /**
    * Parse an unencrypted private key
    *
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 95dd01d2d..ea72b62bf 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -136,6 +136,14 @@ public static String getAccountURL() throws Exception {
     return Utils.constructAccountUrl(scheme, host, port);
   }
 
+  public static String getRole() throws Exception {
+    if (profile == null) {
+      init();
+    }
+
+    return role;
+  }
+
   public static String getWarehouse() throws Exception {
     if (profile == null) {
       init();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 351e2457d..74479abd5 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -1,9 +1,11 @@
 package net.snowflake.ingest.streaming.internal;
 
+import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
 import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
 import static net.snowflake.ingest.utils.Constants.OPEN_CHANNEL_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_ROW_SEQUENCER_IS_COMMITTED;
+import static net.snowflake.ingest.utils.Constants.ROLE;
 import static net.snowflake.ingest.utils.Constants.USER;
 
 import java.security.KeyPair;
@@ -237,7 +239,9 @@ public void testOpenChannelPostRequest() throws Exception {
     Properties prop = new Properties();
     prop.put(USER, TestUtils.getUser());
     prop.put(PRIVATE_KEY, TestUtils.getPrivateKey());
-    prop = Utils.createProperties(prop, false);
+    prop.put(ACCOUNT_URL, TestUtils.getAccountURL());
+    prop.put(ROLE, TestUtils.getRole());
+    prop = Utils.createProperties(prop);
 
     String urlStr = "https://sfctest0.snowflakecomputing.com:80";
     SnowflakeURL url = new SnowflakeURL(urlStr);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index cd62eb144..3a27c0339 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -415,8 +415,9 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
     Properties prop = new Properties();
     prop.put(USER, TestUtils.getUser());
     prop.put(PRIVATE_KEY, TestUtils.getPrivateKey());
-    prop.put(ROLE, "role");
-    prop = Utils.createProperties(prop, false);
+    prop.put(ACCOUNT_URL, TestUtils.getAccountURL());
+    prop.put(ROLE, TestUtils.getRole());
+    prop = Utils.createProperties(prop);
 
     String urlStr = "https://sfctest0.snowflakecomputing.com:80";
     SnowflakeURL url = new SnowflakeURL(urlStr);

From 6b856df7b73b0200a78a6b9f3a3de57d25a56b58 Mon Sep 17 00:00:00 2001
From: Jay Patel <jay.patel@snowflake.com>
Date: Mon, 9 May 2022 14:33:15 -0700
Subject: [PATCH 031/356] no-snow Add readme and call offsetToken API in
 Example file (#177)

* no-snow Add readme and call offsetToken API in Example file

* Review comments
---
 README.md                                     | 19 ++++++-
 .../SnowflakeStreamingIngestExample.java      | 49 +++++++++++++++----
 2 files changed, 58 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 17e635442..78c4c88ed 100644
--- a/README.md
+++ b/README.md
@@ -28,7 +28,7 @@ Documentation Page](docs.snowflake.net).
 ## Maven (Developers only)
 
 This SDK is developed as a [Maven](maven.apache.org) project. As a
-result, you\'ll need to install maven to build the projects and, run
+result, you'll need to install maven to build the projects and, run
 tests.
 
 # Adding as a Dependency
@@ -101,3 +101,20 @@ We use [Google Java format](https://github.com/google/google-java-format) to for
 ```bash
 ./format.sh
 ````
+
+# Snowpipe Streaming Example (Still in Preview)
+
+Run File `SnowflakeStreamingIngestExample.java` which performs following operations.
+1. Reads a JSON file which contains details regarding Snowflake Account, User, Role and Private Key. Take a look at `profile_streaming.json.example` for more details.
+   1. [Here](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication) are the steps required to generate a private key.
+2. Creates a `SnowflakeStreamingIngestClient` which can be used to open one or more Streaming Channels against a table.
+3. Creates a `SnowflakeStreamingIngestChannel` against a Database, Schema and Table name.
+   1. Please note: A Table is expected to be present before opening a Channel. Use following SQL queries and place respective Database, Schema and Table names in `profile_streaming.json` file
+```sql
+create or replace database MY_DATABASE;
+create or replace schema MY_SCHEMA;
+create or replace table MY_TABLE(c1 number);
+```
+4. Inserts a few rows (1000) into a channel created in 3rd step using the `insertRows` API on the Channel object
+   1. `insertRows` API also takes in an optional `offsetToken` String which can be associated to this batch of rows. 
+5. Calls `getLatestCommittedOffsetToken` on the channel until the appropriate offset is found in Snowflake.
\ No newline at end of file
diff --git a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
index 1ce845d01..c1d065241 100644
--- a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
+++ b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
@@ -18,9 +18,13 @@
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
 
-/** Examples on how to use the Streaming Ingest client APIs */
+/**
+ * Example on how to use the Streaming Ingest client APIs.
+ *
+ * <p>Please read the README.md file for detailed steps
+ */
 public class SnowflakeStreamingIngestExample {
-  private static String PROFILE_PATH = "profile.json";
+  private static String PROFILE_PATH = "profile_streaming.json";
   private static final ObjectMapper mapper = new ObjectMapper();
 
   public static void main(String[] args) throws Exception {
@@ -35,28 +39,55 @@ public static void main(String[] args) throws Exception {
     // Create a streaming ingest client
     try (SnowflakeStreamingIngestClient client =
         SnowflakeStreamingIngestClientFactory.builder("CLIENT").setProperties(props).build()) {
+
       // Create an open channel request on table T_STREAMINGINGEST
       OpenChannelRequest request1 =
-          OpenChannelRequest.builder("CHANNEL")
-              .setDBName("DB_STREAMINGINGEST")
-              .setSchemaName("SCHEMA_STREAMINGINGEST")
-              .setTableName("T_STREAMINGINGEST")
+          OpenChannelRequest.builder("MY_CHANNEL")
+              .setDBName("MY_DATABASE")
+              .setSchemaName("MY_SCHEMA")
+              .setTableName("MY_TABLE")
               .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
               .build();
 
       // Open a streaming ingest channel from the given client
       SnowflakeStreamingIngestChannel channel1 = client.openChannel(request1);
 
-      // Insert a few rows into the channel,
-      for (int val = 0; val < 1000; val++) {
+      // Insert few rows into the channel (Using insertRows API), along with the offset Token which
+      // corresponds to the row number
+      final int totalRowsInTable = 1000;
+      for (int val = 0; val < totalRowsInTable; val++) {
         Map<String, Object> row = new HashMap<>();
+
+        // c1 corresponds to the column name in table
         row.put("c1", val);
-        InsertValidationResponse response = channel1.insertRow(row, null);
+
+        InsertValidationResponse response = channel1.insertRow(row, String.valueOf(val));
         if (response.hasErrors()) {
           // Simply throw if there is an exception
           throw response.getInsertErrors().get(0).getException();
         }
       }
+
+      // Polling Snowflake to fetch offset token registered in Snowflake
+      final int expectedOffsetTokenInSnowflake = 999; // because it goes from 0 to 999
+      String offsetTokenFromSnowflake = channel1.getLatestCommittedOffsetToken();
+
+      final int maxRetries = 10;
+      int retryCount = 0;
+      while (offsetTokenFromSnowflake == null
+          || !offsetTokenFromSnowflake.equals(String.valueOf(expectedOffsetTokenInSnowflake))) {
+        Thread.sleep(1_000);
+        offsetTokenFromSnowflake = channel1.getLatestCommittedOffsetToken();
+        retryCount++;
+        if (retryCount >= maxRetries) {
+          System.out.println(
+              String.format(
+                  "Failed to look for required OffsetToken in Snowflake:%s after MaxRetryCounts:%s",
+                  expectedOffsetTokenInSnowflake, maxRetries));
+          System.exit(1);
+        }
+      }
+      System.out.println("SUCCESSFULLY inserted " + totalRowsInTable + " rows");
     }
   }
 }

From f4dc017defbcdb0a8d4f35930d041bc846452620 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 18 May 2022 13:48:47 -0700
Subject: [PATCH 032/356] NO-SNOW: Update Arrow library to 7.0.0 to support
 VectorSchemaRootAppender (#180)

[Apache Arrow library 7.0.0](https://arrow.apache.org/blog/2022/02/08/7.0.0-release/) and plus now supports VectorSchemaRootAppender with BitVector now, switch to the latest 8.0.0 so we could use VectorSchemaRootAppender to append vector to another vector
---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 18d831eec..d92c06934 100644
--- a/pom.xml
+++ b/pom.xml
@@ -319,12 +319,12 @@
         <dependency>
             <groupId>org.apache.arrow</groupId>
             <artifactId>arrow-vector</artifactId>
-            <version>4.0.0</version>
+            <version>8.0.0</version>
         </dependency>
         <dependency>
             <groupId>org.apache.arrow</groupId>
             <artifactId>arrow-memory-netty</artifactId>
-            <version>4.0.0</version>
+            <version>8.0.0</version>
             <scope>runtime</scope>
         </dependency>
 

From 3cd53c91c0ab40ba3cd0736010190a3f9846adcf Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Mon, 6 Jun 2022 21:48:38 -0700
Subject: [PATCH 033/356] SNOW-592855, SNOW-606515: fix channel.close time out
 issue due to high upload latency (#182)

1. Update a few important log lines from DEBUG level to INFO level
2. It's possible that the get status request return an invalid channel status code, so we want to throw as well instead of returning the offset token that may not make sense
3. A bunch of telemetries were moved to JMX, add it back since it's easier to see it in the console
4. Increase the timeout for channel.close() due to high file upload tail latency, we will trust the server side more at this point instead of failing fast
---
 .../internal/ChannelsStatusRequest.java       | 12 ----
 .../streaming/internal/FlushService.java      | 11 ++-
 .../streaming/internal/RegisterService.java   | 37 +++++++---
 ...owflakeStreamingIngestChannelInternal.java | 26 ++++---
 ...nowflakeStreamingIngestClientInternal.java | 68 +++++++++++--------
 .../net/snowflake/ingest/utils/Constants.java |  8 +--
 .../net/snowflake/ingest/utils/ErrorCode.java |  5 +-
 .../ingest/ingest_error_messages.properties   |  1 +
 .../SnowflakeStreamingIngestChannelTest.java  | 15 +++-
 9 files changed, 112 insertions(+), 71 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelsStatusRequest.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelsStatusRequest.java
index 8ae7bf54f..b98782ab9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelsStatusRequest.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelsStatusRequest.java
@@ -4,8 +4,6 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import java.util.List;
 
@@ -29,16 +27,12 @@ static class ChannelStatusRequestDTO {
     // Client Sequencer
     private final Long clientSequencer;
 
-    // Optional Row Sequencer
-    private final Long rowSequencer;
-
     ChannelStatusRequestDTO(SnowflakeStreamingIngestChannelInternal channel) {
       this.channelName = channel.getName();
       this.databaseName = channel.getDBName();
       this.schemaName = channel.getSchemaName();
       this.tableName = channel.getTableName();
       this.clientSequencer = channel.getChannelSequencer();
-      this.rowSequencer = channel.getRowSequencer();
     }
 
     @JsonProperty("table")
@@ -65,12 +59,6 @@ String getChannelName() {
     Long getClientSequencer() {
       return clientSequencer;
     }
-
-    @JsonInclude(Include.NON_EMPTY)
-    @JsonProperty("row_sequencer")
-    Long getRowSequencer() {
-      return rowSequencer;
-    }
   }
 
   // Optional Request ID. Used for diagnostic purposes.
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 7a9f43e94..0a49a86b9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -525,7 +525,7 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
         curDataSize += encryptedCompressedChunkDataSize;
         crc.update(encryptedCompressedChunkData, 0, encryptedCompressedChunkDataSize);
 
-        logger.logDebug(
+        logger.logInfo(
             "Finish building chunk in blob={}, table={}, rowCount={}, uncompressedSize={},"
                 + " compressedChunkLength={}, encryptedCompressedSize={}",
             filePath,
@@ -558,7 +558,8 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
    */
   BlobMetadata upload(String filePath, byte[] blob, List<ChunkMetadata> metadata)
       throws NoSuchAlgorithmException {
-    logger.logDebug("Start uploading file={}, size={}", filePath, blob.length);
+    logger.logInfo("Start uploading file={}, size={}", filePath, blob.length);
+    long startTime = System.currentTimeMillis();
 
     Timer.Context uploadContext = Utils.createTimerContext(this.owningClient.uploadLatency);
 
@@ -572,7 +573,11 @@ BlobMetadata upload(String filePath, byte[] blob, List<ChunkMetadata> metadata)
           metadata.stream().mapToLong(i -> i.getEpInfo().getRowCount()).sum());
     }
 
-    logger.logDebug("Finish uploading file={}", filePath);
+    logger.logInfo(
+        "Finish uploading file={}, size={}, timeInMillis={}",
+        filePath,
+        blob.length,
+        System.currentTimeMillis() - startTime);
 
     return new BlobMetadata(filePath, BlobBuilder.computeMD5(blob), metadata);
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index fa663d517..bbcd2fecf 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -75,7 +75,7 @@ void addBlobs(List<Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>>>
    * the ordering is maintained across independent blobs in the same channel.
    *
    * @param latencyTimerContextMap the map that stores the latency timer for each blob
-   * @return a list of blob names that has errors during registration
+   * @return a list of blob names that have errors during registration
    */
   List<FlushService.BlobData> registerBlobs(Map<String, Timer.Context> latencyTimerContextMap) {
     List<FlushService.BlobData> errorBlobs = new ArrayList<>();
@@ -92,15 +92,26 @@ List<FlushService.BlobData> registerBlobs(Map<String, Timer.Context> latencyTime
           this.blobsListLock.unlock();
         }
 
-        // If no exception, we will register all blobs in the blob list
-        // If hitting non-timeout exception, we will log it and continue
-        // If hitting timeout exception, we will register all the previous blobs in the blob list
-        // and retry until the hitting BLOB_UPLOAD_MAX_RETRY_COUNT
+        // In order to guarantee fairness between the time spent on waiting blob uploading VS blob
+        // registering and make sure the delay on server side commit is relatively small:
+        // 1. If no exception and total time is less than or equal to
+        // BLOB_UPLOAD_TIMEOUT_IN_SEC * 2, we will wait for all blobs to be uploaded and then
+        // register them
+        // 2. If no exception and total time is bigger than BLOB_UPLOAD_TIMEOUT_IN_SEC * 2, we
+        // will break the waiting and register the processed blob first
+        // 3. If hitting non-timeout exception, we will skip the current blob and continue on
+        // processing next blob
+        // 4. If hitting timeout exception, we will break the waiting if we haven't reached
+        // BLOB_UPLOAD_TIMEOUT_IN_SEC * BLOB_UPLOAD_MAX_RETRY_COUNT and register the
+        // processed blob first. Otherwise, we will skip the current blob and continue on processing
+        // next blob if time out has been reached.
         int idx = 0;
         int retry = 0;
         while (idx < oldList.size()) {
           List<BlobMetadata> blobs = new ArrayList<>();
-          while (idx < oldList.size()) {
+          long startTime = System.currentTimeMillis();
+          while (idx < oldList.size()
+              && System.currentTimeMillis() - startTime <= BLOB_UPLOAD_TIMEOUT_IN_SEC * 2) {
             Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>> futureBlob =
                 oldList.get(idx);
             try {
@@ -124,12 +135,14 @@ List<FlushService.BlobData> registerBlobs(Map<String, Timer.Context> latencyTime
 
               // Retry logic for timeout exception only
               if (e instanceof TimeoutException && retry < BLOB_UPLOAD_MAX_RETRY_COUNT) {
+                logger.logInfo(
+                    "Retry on waiting for uploading blob={}", futureBlob.getKey().getFilePath());
                 retry++;
                 break;
               }
               logger.logError(
-                  "Building or uploading blob failed={}, exception={}, detail={}, cause={},"
-                      + " detail={}, all channels in the blob will be invalidated",
+                  "Building or uploading blob={} failed or timed out, exception={}, detail={},"
+                      + " cause={}, detail={}, all channels in the blob will be invalidated",
                   futureBlob.getKey().getFilePath(),
                   e,
                   e.getMessage(),
@@ -145,10 +158,16 @@ List<FlushService.BlobData> registerBlobs(Map<String, Timer.Context> latencyTime
           }
 
           if (blobs.size() > 0 && !isTestMode) {
+            logger.logInfo(
+                "Start to registering blobs in client={}, totalBlobListSize={},"
+                    + " currentBlobListSize={}",
+                this.owningClient.getName(),
+                oldList.size(),
+                blobs.size());
             Timer.Context registerContext =
                 Utils.createTimerContext(this.owningClient.registerLatency);
 
-            // Register the blobs, and invalidate any channels that returns a failure status code
+            // Register the blobs, and invalidate any channels that return a failure status code
             this.owningClient.registerBlobs(blobs);
 
             if (registerContext != null) {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 47b15d878..16022ad98 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -6,6 +6,7 @@
 
 import static net.snowflake.ingest.utils.Constants.INSERT_THROTTLE_MAX_RETRY_COUNT;
 import static net.snowflake.ingest.utils.Constants.MAX_CHUNK_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
 
 import java.util.Collections;
 import java.util.List;
@@ -112,7 +113,7 @@ class SnowflakeStreamingIngestChannelInternal implements SnowflakeStreamingInges
     this.encryptionKey = encryptionKey;
     this.encryptionKeyId = encryptionKeyId;
     this.onErrorOption = onErrorOption;
-    logger.logDebug("Channel={} created for table={}", this.channelName, this.tableName);
+    logger.logInfo("Channel={} created for table={}", this.channelName, this.tableName);
   }
 
   /**
@@ -264,8 +265,8 @@ public boolean isClosed() {
   /** Mark the channel as closed */
   void markClosed() {
     this.isClosed = true;
-    logger.logDebug(
-        "Channel is closed, name={}, channel sequencer={}, row sequencer={}",
+    logger.logInfo(
+        "Channel is marked as closed, name={}, channel sequencer={}, row sequencer={}",
         getFullyQualifiedName(),
         channelSequencer,
         rowSequencer);
@@ -317,10 +318,10 @@ public CompletableFuture<Void> close() {
               this.arrowBuffer.close();
               this.owningClient.removeChannelIfSequencersMatch(this);
 
-              // Throw an exception if the channel has any uncommitted rows
-              if (!uncommittedChannels.isEmpty()) {
+              // Throw an exception if the channel is invalid or has any uncommitted rows
+              if (!isValid() || !uncommittedChannels.isEmpty()) {
                 throw new SFException(
-                    ErrorCode.CHANNEL_WITH_UNCOMMITTED_ROWS,
+                    ErrorCode.CHANNELS_WITH_UNCOMMITTED_ROWS,
                     uncommittedChannels.stream()
                         .map(SnowflakeStreamingIngestChannelInternal::getFullyQualifiedName)
                         .collect(Collectors.toList()));
@@ -419,11 +420,14 @@ void collectRowSize(float rowSize) {
   public String getLatestCommittedOffsetToken() {
     checkValidation();
 
-    return this.owningClient
-        .getChannelsStatus(Collections.singletonList(this))
-        .getChannels()
-        .get(0)
-        .getPersistedOffsetToken();
+    ChannelsStatusResponse.ChannelStatusResponseDTO response =
+        this.owningClient.getChannelsStatus(Collections.singletonList(this)).getChannels().get(0);
+
+    if (response.getStatusCode() != RESPONSE_SUCCESS) {
+      throw new SFException(ErrorCode.CHANNEL_STATUS_INVALID, getName(), response.getStatusCode());
+    }
+
+    return response.getPersistedOffsetToken();
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index f4abef7c3..4c695c82b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -18,7 +18,6 @@
 import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_ROW_SEQUENCER_IS_COMMITTED;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
 import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JMX_METRIC_PREFIX;
 import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JVM_MEMORY_AND_THREAD_METRICS_REGISTRY;
@@ -180,7 +179,7 @@ public class SnowflakeStreamingIngestClientInternal implements SnowflakeStreamin
       this.registerMetricsForClient();
     }
 
-    logger.logDebug(
+    logger.logInfo(
         "Client created, name={}, account={}. isTestMode={}, parameters={}",
         name,
         accountURL == null ? "" : accountURL.getAccount(),
@@ -374,8 +373,8 @@ void registerBlobs(List<BlobMetadata> blobs) {
    * @param executionCount Number of times this call has been attempted, used to track retries
    */
   void registerBlobs(List<BlobMetadata> blobs, final int executionCount) {
-    logger.logDebug(
-        "Register blob request start for blob={}, client={}, executionCount={}",
+    logger.logInfo(
+        "Register blob request preparing for blob={}, client={}, executionCount={}",
         blobs.stream().map(BlobMetadata::getPath).collect(Collectors.toList()),
         this.name,
         executionCount);
@@ -412,8 +411,8 @@ void registerBlobs(List<BlobMetadata> blobs, final int executionCount) {
       throw new SFException(e, ErrorCode.REGISTER_BLOB_FAILURE, e.getMessage());
     }
 
-    logger.logDebug(
-        "Register blob request succeeded for blob={}, client={}, executionCount={}",
+    logger.logInfo(
+        "Register blob request returned for blob={}, client={}, executionCount={}",
         blobs.stream().map(BlobMetadata::getPath).collect(Collectors.toList()),
         this.name,
         executionCount);
@@ -461,7 +460,7 @@ void registerBlobs(List<BlobMetadata> blobs, final int executionCount) {
                                     })));
 
     if (!queueFullChunks.isEmpty()) {
-      logger.logDebug(
+      logger.logInfo(
           "Retrying registerBlobs request, blobs={}, retried_chunks={}, executionCount={}",
           blobs,
           queueFullChunks,
@@ -512,10 +511,10 @@ List<BlobMetadata> getRetryBlobs(
                           chunkMetadata.getChannels().stream()
                               .map(
                                   channelMetadata ->
-                                      new Pair(
+                                      new Pair<>(
                                           channelMetadata.getChannelName(),
                                           channelMetadata.getClientSequencer()))
-                              .anyMatch(channelKey -> queueFullKeys.contains(channelKey)))
+                              .anyMatch(queueFullKeys::contains))
                   .collect(Collectors.toList());
           if (!relevantChunks.isEmpty()) {
             retryBlobs.add(
@@ -536,20 +535,21 @@ public void close() throws Exception {
     isClosed = true;
     this.channelCache.closeAllChannels();
 
-    // unregister jmx metrics
-    if (this.metrics != null) {
-      removeMetricsFromRegistry();
-
-      // LOG jvm memory and thread metrics at the end
-      Slf4jReporter.forRegistry(jvmMemoryAndThreadMetrics)
-          .outputTo(logger.getLogger())
-          .build()
-          .report();
-    }
-
     // Flush any remaining rows and cleanup all the resources
     try {
       this.flush(true).get();
+
+      // unregister jmx metrics
+      if (this.metrics != null) {
+        Slf4jReporter.forRegistry(metrics).outputTo(logger.getLogger()).build().report();
+        removeMetricsFromRegistry();
+
+        // LOG jvm memory and thread metrics at the end
+        Slf4jReporter.forRegistry(jvmMemoryAndThreadMetrics)
+            .outputTo(logger.getLogger())
+            .build()
+            .report();
+      }
     } catch (InterruptedException | ExecutionException e) {
       throw new SFException(e, ErrorCode.RESOURCE_CLEANUP_FAILURE, "client close");
     } finally {
@@ -631,6 +631,7 @@ List<SnowflakeStreamingIngestChannelInternal> verifyChannelsAreFullyCommitted(
     int retry = 0;
     boolean isTimeout = true;
     List<ChannelsStatusResponse.ChannelStatusResponseDTO> oldChannelsStatus = new ArrayList<>();
+    List<SnowflakeStreamingIngestChannelInternal> channelsWithError = new ArrayList<>();
     do {
       List<ChannelsStatusResponse.ChannelStatusResponseDTO> channelsStatus =
           getChannelsStatus(channels).getChannels();
@@ -638,9 +639,23 @@ List<SnowflakeStreamingIngestChannelInternal> verifyChannelsAreFullyCommitted(
       List<ChannelsStatusResponse.ChannelStatusResponseDTO> tempChannelsStatus = new ArrayList<>();
 
       for (int idx = 0; idx < channelsStatus.size(); idx++) {
-        if (channelsStatus.get(idx).getStatusCode() != RESPONSE_ROW_SEQUENCER_IS_COMMITTED) {
-          tempChannels.add(channels.get(idx));
-          tempChannelsStatus.add(channelsStatus.get(idx));
+        ChannelsStatusResponse.ChannelStatusResponseDTO channelStatus = channelsStatus.get(idx);
+        SnowflakeStreamingIngestChannelInternal channel = channels.get(idx);
+        logger.logInfo(
+            "Get channel status name={}, status={}, clientSequencer={}, rowSequencer={},"
+                + " offsetToken={}, persistedRowSequencer={}, persistedOffsetToken={}",
+            channel.getName(),
+            channelStatus.getStatusCode(),
+            channel.getChannelSequencer(),
+            channel.getRowSequencer(),
+            channel.getOffsetToken(),
+            channelStatus.getPersistedRowSequencer(),
+            channelStatus.getPersistedOffsetToken());
+        if (channelStatus.getStatusCode() != RESPONSE_SUCCESS) {
+          channelsWithError.add(channel);
+        } else if (!channelStatus.getPersistedRowSequencer().equals(channel.getRowSequencer())) {
+          tempChannels.add(channel);
+          tempChannelsStatus.add(channelStatus);
         }
       }
 
@@ -685,6 +700,7 @@ List<SnowflakeStreamingIngestChannelInternal> verifyChannelsAreFullyCommitted(
           this.name);
     }
 
+    channels.addAll(channelsWithError);
     return channels;
   }
 
@@ -753,11 +769,9 @@ private void registerMetricsForClient() {
    */
   static ObjectName getObjectName(String clientName, String jmxDomain, String metricName) {
     try {
-      StringBuilder sb = new StringBuilder(jmxDomain).append(":clientName=").append(clientName);
-
-      sb.append(",name=").append(metricName);
+      String sb = jmxDomain + ":clientName=" + clientName + ",name=" + metricName;
 
-      return new ObjectName(sb.toString());
+      return new ObjectName(sb);
     } catch (MalformedObjectNameException e) {
       logger.logWarn("Could not create Object name for MetricName={}", metricName);
       throw new SFException(ErrorCode.INTERNAL_ERROR, "Invalid metric name");
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index 437f7f463..bd935ec36 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -28,7 +28,7 @@ public class Constants {
       10L; // Don't change, should match server side
   public static final long RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL =
       7L; // Don't change, should match server side
-  public static final long BLOB_UPLOAD_TIMEOUT_IN_SEC = 5L;
+  public static final int BLOB_UPLOAD_TIMEOUT_IN_SEC = 5;
   public static final int BLOB_UPLOAD_MAX_RETRY_COUNT = 12;
   public static final int INSERT_THROTTLE_MAX_RETRY_COUNT = 10;
   public static final long MAX_BLOB_SIZE_IN_BYTES = 512000000L;
@@ -44,10 +44,8 @@ public class Constants {
   public static final int MAX_THREAD_COUNT = Integer.MAX_VALUE;
   public static final int CPU_IO_TIME_RATIO = 1;
   public static final String CLIENT_CONFIGURE_ENDPOINT = "/v1/streaming/client/configure/";
-  public static final int COMMIT_MAX_RETRY_COUNT = 20;
-  public static final int COMMIT_RETRY_INTERVAL_IN_MS = 500;
-  public static final int RESPONSE_ROW_SEQUENCER_IS_COMMITTED =
-      26; // Don't change, should match server side
+  public static final int COMMIT_MAX_RETRY_COUNT = 60;
+  public static final int COMMIT_RETRY_INTERVAL_IN_MS = 1000;
   public static final String ENCRYPTION_ALGORITHM = "AES/CTR/NoPadding";
   public static final int ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES = 16;
   public static final int MAX_STREAMING_INGEST_API_CHANNEL_RETRY = 3;
diff --git a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
index 68ab2b569..095d08d14 100644
--- a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
+++ b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
@@ -30,9 +30,10 @@ public enum ErrorCode {
   KEYPAIR_CREATION_FAILURE("0022"),
   MD5_HASHING_NOT_AVAILABLE("0023"),
   CHANNEL_STATUS_FAILURE("0024"),
-  CHANNEL_WITH_UNCOMMITTED_ROWS("0025"),
+  CHANNELS_WITH_UNCOMMITTED_ROWS("0025"),
   INVALID_COLLATION_STRING("0026"),
-  ENCRYPTION_FAILURE("0027");
+  ENCRYPTION_FAILURE("0027"),
+  CHANNEL_STATUS_INVALID("0028");
 
   public static final String errorMessageResource = "net.snowflake.ingest.ingest_error_messages";
 
diff --git a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
index 550b3355a..ef0149ce4 100644
--- a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
+++ b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
@@ -30,5 +30,6 @@
 0025=One or more channels {0} might contain uncommitted rows due to server side errors, please consider reopening the channels to replay the data loading by using the latest persistent offset token.
 0026=Invalid collation string: {0}. {1}
 0027=Failure during data encryption.
+0028=Get channel status indicates Channel {0} is invalid with status code {1}, please reopening the channel.
 
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 74479abd5..a68f5db7b 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -4,7 +4,7 @@
 import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
 import static net.snowflake.ingest.utils.Constants.OPEN_CHANNEL_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_ROW_SEQUENCER_IS_COMMITTED;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
 import static net.snowflake.ingest.utils.Constants.ROLE;
 import static net.snowflake.ingest.utils.Constants.USER;
 
@@ -648,14 +648,25 @@ public void testGetLatestCommittedOffsetToken() {
     ChannelsStatusResponse response = new ChannelsStatusResponse();
     response.setStatusCode(0L);
     response.setMessage("Success");
+
+    // Test success case
     ChannelsStatusResponse.ChannelStatusResponseDTO channelStatus =
         new ChannelsStatusResponse.ChannelStatusResponseDTO();
-    channelStatus.setStatusCode((long) RESPONSE_ROW_SEQUENCER_IS_COMMITTED);
+    channelStatus.setStatusCode((long) RESPONSE_SUCCESS);
     channelStatus.setPersistedOffsetToken(offsetToken);
     response.setChannels(Collections.singletonList(channelStatus));
 
     Mockito.doReturn(response).when(client).getChannelsStatus(Mockito.any());
 
     Assert.assertEquals(offsetToken, channel.getLatestCommittedOffsetToken());
+
+    // Test error case
+    channelStatus.setStatusCode(-1L);
+    try {
+      Assert.assertEquals(offsetToken, channel.getLatestCommittedOffsetToken());
+      Assert.fail("Get offset token should fail");
+    } catch (SFException e) {
+      Assert.assertEquals(ErrorCode.CHANNEL_STATUS_INVALID.getMessageCode(), e.getVendorCode());
+    }
   }
 }

From 093339bed5b9834a4f2bcdf47f85880fe7e28909 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Tue, 7 Jun 2022 13:26:52 -0700
Subject: [PATCH 034/356] NO-SNOW: fix test failure due to time zone difference
 (#183)

Explicitly set the timezone to be UTC for NTZ so the test won't fail when running on non-UTC environment
---
 .../streaming/internal/StreamingIngestIT.java | 26 ++++++++++++-------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index f50e12c9b..5c16b7b78 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -9,10 +9,12 @@
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Calendar;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.TimeZone;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -74,6 +76,8 @@ public void beforeAll() throws Exception {
             String.format(
                 "alter table %s set ENABLE_PR_37692_MULTI_FORMAT_SCANSET=true;", TEST_TABLE));
     jdbcConnection.createStatement().execute("alter session set ENABLE_UNIFIED_TABLE_SCAN=true;");
+    // Set timezone to UTC
+    jdbcConnection.createStatement().execute("alter session set timezone = 'UTC';");
     jdbcConnection
         .createStatement()
         .execute(String.format("use warehouse %s", TestUtils.getWarehouse()));
@@ -387,6 +391,8 @@ public void testTimeColumnIngest() throws Exception {
     // Close the channel after insertion
     channel1.close().get();
 
+    Calendar cal = Calendar.getInstance();
+    cal.setTimeZone(TimeZone.getTimeZone("UTC"));
     for (int i = 1; i < 15; i++) {
       if (channel1.getLatestCommittedOffsetToken() != null
           && channel1.getLatestCommittedOffsetToken().equals("1")) {
@@ -404,9 +410,9 @@ public void testTimeColumnIngest() throws Exception {
         Assert.assertEquals(3600123, result.getTimestamp("TSMALL").getTime());
         Assert.assertEquals(32400123, result.getTimestamp("TBIG").getTime());
         Assert.assertEquals(123456780, result.getTimestamp("TBIG").getNanos());
-        Assert.assertEquals(1609462800123L, result.getTimestamp("TNTZSMALL").getTime());
-        Assert.assertEquals(1609462800123L, result.getTimestamp("TNTZBIG").getTime());
-        Assert.assertEquals(123450000, result.getTimestamp("TNTZBIG").getNanos());
+        Assert.assertEquals(1609462800123L, result.getTimestamp("TNTZSMALL", cal).getTime());
+        Assert.assertEquals(1609462800123L, result.getTimestamp("TNTZBIG", cal).getTime());
+        Assert.assertEquals(123450000, result.getTimestamp("TNTZBIG", cal).getNanos());
 
         result =
             jdbcConnection
@@ -430,9 +436,9 @@ public void testTimeColumnIngest() throws Exception {
         Assert.assertEquals(10800123, result.getTimestamp("MTSMALL").getTime());
         Assert.assertEquals(39600123, result.getTimestamp("MTBIG").getTime());
         Assert.assertEquals(123456780, result.getTimestamp("MTBIG").getNanos());
-        Assert.assertEquals(1809462800123L, result.getTimestamp("MTNTZSMALL").getTime());
-        Assert.assertEquals(1925024400123L, result.getTimestamp("MTNTZBIG").getTime());
-        Assert.assertEquals(123456780, result.getTimestamp("MTNTZBIG").getNanos());
+        Assert.assertEquals(1809462800123L, result.getTimestamp("MTNTZSMALL", cal).getTime());
+        Assert.assertEquals(1925024400123L, result.getTimestamp("MTNTZBIG", cal).getTime());
+        Assert.assertEquals(123456780, result.getTimestamp("MTNTZBIG", cal).getNanos());
 
         return;
       } else {
@@ -471,12 +477,14 @@ public void testMultiColumnIngest() throws Exception {
     row.put("tinyfloat", 1.1);
     row.put("var", "{\"e\":2.7}");
     row.put("t", timestamp);
-    row.put("d", "1967-06-23 01:01:01");
+    row.put("d", "1969-12-31 00:00:00");
     verifyInsertValidationResponse(channel1.insertRow(row, "1"));
 
     // Close the channel after insertion
     channel1.close().get();
 
+    Calendar cal = Calendar.getInstance();
+    cal.setTimeZone(TimeZone.getTimeZone("UTC"));
     for (int i = 1; i < 15; i++) {
       if (channel1.getLatestCommittedOffsetToken() != null
           && channel1.getLatestCommittedOffsetToken().equals("1")) {
@@ -493,8 +501,8 @@ public void testMultiColumnIngest() throws Exception {
         Assert.assertEquals(3.14, result.getFloat("F"), 0.0001);
         Assert.assertEquals(1.1, result.getFloat("TINYFLOAT"), 0.001);
         Assert.assertEquals("{\n" + "  \"e\": 2.7\n" + "}", result.getString("VAR"));
-        Assert.assertEquals(timestamp * 1000, result.getTimestamp("T").getTime());
-        Assert.assertEquals(-923, TimeUnit.MILLISECONDS.toDays(result.getDate("D").getTime()));
+        Assert.assertEquals(timestamp * 1000, result.getTimestamp("T", cal).getTime());
+        Assert.assertEquals(-1, TimeUnit.MILLISECONDS.toDays(result.getDate("D", cal).getTime()));
         return;
       } else {
         Thread.sleep(2000);

From c26ba1fbb2c914f933754d6e2c47098257fa126b Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 15 Jun 2022 11:45:11 -0700
Subject: [PATCH 035/356] NO-SNOW: debug memory issue (#185)

We kept having memory management issues in the Arrow buffers during the channel invalidation scenario, this PR contains two major changes:

1. Add more memory related logging to debug memory issue when we hit it again
2. Don't release bytes explicitly to avoid the IllegalArgumentException: Accounted size went negative. issue, the memory should be released with vectorRoot.close
---
 .../streaming/internal/ArrowRowBuffer.java       | 16 ++++++++++++----
 .../SnowflakeStreamingIngestChannelInternal.java | 11 +++++++----
 .../SnowflakeStreamingIngestClientInternal.java  |  2 +-
 .../java/net/snowflake/ingest/utils/Utils.java   |  2 --
 .../ingest/streaming/internal/RowBufferTest.java |  2 +-
 5 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index a3e15d02a..7b8074078 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -220,23 +220,31 @@ void setupSchema(List<ColumnMetadata> columns) {
    * Close the row buffer and release resources. Note that the caller needs to handle
    * synchronization
    */
-  void close() {
-    long allocated = this.allocator.getAllocatedMemory();
+  void close(String name) {
+    long allocatedBeforeRelease = this.allocator.getAllocatedMemory();
     if (this.vectorsRoot != null) {
       this.vectorsRoot.close();
       this.tempVectorsRoot.close();
     }
     this.fields.clear();
+    long allocatedAfterRelease = this.allocator.getAllocatedMemory();
+    logger.logInfo(
+        "Trying to close arrow buffer for channel={} from function={}, allocatedBeforeRelease={},"
+            + " allocatedAfterRelease={}",
+        this.owningChannel.getName(),
+        name,
+        allocatedBeforeRelease,
+        allocatedAfterRelease);
     Utils.closeAllocator(this.allocator);
 
     // If the channel is valid but still has leftover data, throw an exception because it should be
     // cleaned up already before calling close
-    if (allocated > 0 && this.owningChannel.isValid()) {
+    if (allocatedBeforeRelease > 0 && this.owningChannel.isValid()) {
       throw new SFException(
           ErrorCode.INTERNAL_ERROR,
           String.format(
               "Memory leaked=%d by allocator=%s, channel=%s",
-              allocated, this.allocator.toString(), this.owningChannel.getFullyQualifiedName()));
+              allocatedBeforeRelease, this.allocator, this.owningChannel.getFullyQualifiedName()));
     }
   }
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 16022ad98..5eedb2855 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -108,7 +108,10 @@ class SnowflakeStreamingIngestChannelInternal implements SnowflakeStreamingInges
             ? new RootAllocator()
             : this.owningClient
                 .getAllocator()
-                .newChildAllocator(name, 0, this.owningClient.getAllocator().getLimit());
+                .newChildAllocator(
+                    String.format("%s_%s", name, channelSequencer),
+                    0,
+                    this.owningClient.getAllocator().getLimit());
     this.arrowBuffer = new ArrowRowBuffer(this);
     this.encryptionKey = encryptionKey;
     this.encryptionKeyId = encryptionKeyId;
@@ -248,7 +251,7 @@ public boolean isValid() {
   /** Mark the channel as invalid, and release resources */
   void invalidate() {
     this.isValid = false;
-    this.arrowBuffer.close();
+    this.arrowBuffer.close("invalidate");
     logger.logWarn(
         "Channel is invalidated, name={}, channel sequencer={}, row sequencer={}",
         getFullyQualifiedName(),
@@ -315,7 +318,7 @@ public CompletableFuture<Void> close() {
                   this.owningClient.verifyChannelsAreFullyCommitted(
                       Collections.singletonList(this));
 
-              this.arrowBuffer.close();
+              this.arrowBuffer.close("close");
               this.owningClient.removeChannelIfSequencersMatch(this);
 
               // Throw an exception if the channel is invalid or has any uncommitted rows
@@ -473,7 +476,7 @@ void throttleInsertIfNeeded(Runtime runtime) {
   private void checkValidation() {
     if (!isValid()) {
       this.owningClient.removeChannelIfSequencersMatch(this);
-      this.arrowBuffer.close();
+      this.arrowBuffer.close("checkValidation");
       throw new SFException(ErrorCode.INVALID_CHANNEL, getFullyQualifiedName());
     }
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 4c695c82b..f6d97ee2d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -417,7 +417,7 @@ void registerBlobs(List<BlobMetadata> blobs, final int executionCount) {
         this.name,
         executionCount);
 
-    // We will retry any blob chunks that were rejected becuase internal Snowflake queues are full
+    // We will retry any blob chunks that were rejected because internal Snowflake queues are full
     Set<ChunkRegisterStatus> queueFullChunks = new HashSet<>();
     response
         .getBlobsStatus()
diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java
index 5b421f9d1..68ef2124d 100644
--- a/src/main/java/net/snowflake/ingest/utils/Utils.java
+++ b/src/main/java/net/snowflake/ingest/utils/Utils.java
@@ -256,10 +256,8 @@ public static boolean isNullOrEmpty(String string) {
   /** Release any outstanding memory and then close the buffer allocator */
   public static void closeAllocator(BufferAllocator alloc) {
     for (BufferAllocator childAlloc : alloc.getChildAllocators()) {
-      childAlloc.releaseBytes(childAlloc.getAllocatedMemory());
       childAlloc.close();
     }
-    alloc.releaseBytes(alloc.getAllocatedMemory());
     alloc.close();
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 23b36a1b5..6b22c8c85 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -718,7 +718,7 @@ private void testInsertRowsHelper(ArrowRowBuffer rowBuffer) {
 
   @Test
   public void testClose() {
-    this.rowBufferOnErrorContinue.close();
+    this.rowBufferOnErrorContinue.close("testClose");
     Map<String, Object> row = new HashMap<>();
     row.put("colTinyInt", (byte) 1);
     row.put("colSmallInt", (short) 2);

From e47f7b868c722c02345d1022cd6e50ed608bc5fc Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 15 Jun 2022 14:42:40 -0700
Subject: [PATCH 036/356] fix name separator (#187)

We have seen an issue that using the Client SDK under Windows would not work because we're using the wrong name separator with Paths.get, update to use "/" explicitly.
---
 .../ingest/streaming/internal/FlushService.java      | 12 +-----------
 .../streaming/internal/StreamingIngestStage.java     |  1 -
 2 files changed, 1 insertion(+), 12 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 0a49a86b9..85c9c61e4 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -15,8 +15,6 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.security.InvalidAlgorithmParameterException;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
@@ -647,15 +645,7 @@ String getFilePath(Calendar calendar, String clientPrefix) {
             + this.counter.getAndIncrement()
             + "."
             + BLOB_EXTENSION_TYPE;
-    Path filePath =
-        Paths.get(
-            Integer.toString(year),
-            Integer.toString(month),
-            Integer.toString(day),
-            Integer.toString(hour),
-            Integer.toString(minute),
-            fileName);
-    return filePath.toString();
+    return year + "/" + month + "/" + day + "/" + hour + "/" + minute + "/" + fileName;
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index 338c735ad..8a23845b5 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -375,7 +375,6 @@ void putLocal(String fullFilePath, byte[] data) {
     InputStream input = new ByteArrayInputStream(data);
     try {
       String stageLocation = this.fileTransferMetadataWithAge.localLocation;
-      Paths.get(stageLocation, fullFilePath);
       File destFile = Paths.get(stageLocation, fullFilePath).toFile();
       FileUtils.copyInputStreamToFile(input, destFile);
     } catch (Exception ex) {

From 07228e5620037571fdc122d4ef59681db98ecb68 Mon Sep 17 00:00:00 2001
From: Matt Naides <63469689+sfc-gh-mnaides@users.noreply.github.com>
Date: Wed, 15 Jun 2022 15:56:49 -0700
Subject: [PATCH 037/356] SNOW-608483 fix column stats for collated strings
 (#186)

* report col and noncol strings as the same

* cleanup and fix test

* remove redundant field from RowBufferStats

* add comment

* fix format
---
 .../internal/FileColumnProperties.java        |  7 ++-
 .../streaming/internal/RowBufferStats.java    | 56 +------------------
 .../streaming/internal/ChannelDataTest.java   |  8 +--
 .../internal/RowBufferStatsTest.java          | 48 ++++------------
 .../streaming/internal/RowBufferTest.java     |  8 +--
 5 files changed, 28 insertions(+), 99 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java b/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java
index 9322a44ff..ecc6dc031 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java
@@ -51,8 +51,11 @@ class FileColumnProperties {
     this.setMinRealValue(stats.getCurrentMinRealValue());
     this.setMaxRealValue(stats.getCurrentMaxRealValue());
     this.setMaxLength(stats.getCurrentMaxLength());
-    this.setMaxStrNonCollated(stats.getCurrentMaxStrValue());
-    this.setMinStrNonCollated(stats.getCurrentMinStrValue());
+
+    // Collated and non-collated strings are intentionally equal here as required by Snowflake
+    this.setMaxStrNonCollated(stats.getCurrentMaxColStrValue());
+    this.setMinStrNonCollated(stats.getCurrentMinColStrValue());
+
     this.setMaxStrValue(stats.getCurrentMaxColStrValue());
     this.setMinStrValue(stats.getCurrentMinColStrValue());
     this.setNullCount(stats.getCurrentNullCount());
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
index f490671a2..6aaad8e6f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
@@ -296,14 +296,10 @@ byte[] performConversion(String input) {
     }
   }
 
-  private String currentMinStrValue;
-  private String currentMaxStrValue;
   private String currentMinColStrValue;
   private String currentMaxColStrValue;
   private byte[] currentMinColStrValueInBytes;
   private byte[] currentMaxColStrValueInBytes;
-  private byte[] currentMinStrValueInBytes;
-  private byte[] currentMaxStrValueInBytes;
   private BigInteger currentMinIntValue;
   private BigInteger currentMaxIntValue;
   private Double currentMinRealValue;
@@ -330,14 +326,10 @@ byte[] performConversion(String input) {
   }
 
   void reset() {
-    this.currentMaxStrValue = null;
-    this.currentMinStrValue = null;
     this.currentMaxColStrValue = null;
     this.currentMinColStrValue = null;
     this.currentMaxColStrValueInBytes = null;
     this.currentMinColStrValueInBytes = null;
-    this.currentMinStrValueInBytes = null;
-    this.currentMaxStrValueInBytes = null;
     this.currentMaxIntValue = null;
     this.currentMinIntValue = null;
     this.currentMaxRealValue = null;
@@ -375,16 +367,12 @@ static RowBufferStats getCombinedStats(RowBufferStats left, RowBufferStats right
       combined.addIntValue(right.currentMaxIntValue);
     }
 
-    if (left.currentMinStrValue != null) {
-      combined.addStrValue(left.currentMinStrValue);
-      combined.addStrValue(left.currentMaxStrValue);
+    if (left.currentMinColStrValue != null) {
       combined.addStrValue(left.currentMinColStrValue);
       combined.addStrValue(left.currentMaxColStrValue);
     }
 
-    if (right.currentMinStrValue != null) {
-      combined.addStrValue(right.currentMinStrValue);
-      combined.addStrValue(right.currentMaxStrValue);
+    if (right.currentMinColStrValue != null) {
       combined.addStrValue(right.currentMinColStrValue);
       combined.addStrValue(right.currentMaxColStrValue);
     }
@@ -414,10 +402,8 @@ void addStrValue(String inputValue) {
     byte[] collatedValueBytes = value != null ? getCollatedBytes(value) : null;
 
     // Check if new min/max string
-    if (this.currentMinStrValue == null) {
-      this.currentMinStrValue = value;
+    if (this.currentMinColStrValue == null) {
       this.currentMinColStrValue = value;
-      this.currentMinStrValueInBytes = valueBytes;
       this.currentMinColStrValueInBytes = collatedValueBytes;
 
       /*
@@ -431,41 +417,13 @@ void addStrValue(String inputValue) {
         incrementedValueBytes[MAX_LOB_LEN - 1]++;
         incrementedCollatedValueBytes[MAX_LOB_LEN - 1]++;
         String incrementedValue = new String(incrementedValueBytes);
-        this.currentMaxStrValue = incrementedValue;
         this.currentMaxColStrValue = incrementedValue;
-        this.currentMaxStrValueInBytes = incrementedValueBytes;
         this.currentMaxColStrValueInBytes = incrementedCollatedValueBytes;
       } else {
-        this.currentMaxStrValue = value;
         this.currentMaxColStrValue = value;
-        this.currentMaxStrValueInBytes = valueBytes;
         this.currentMaxColStrValueInBytes = collatedValueBytes;
       }
     } else {
-      // Non-collated comparison
-      if (compare(currentMinStrValueInBytes, valueBytes) > 0) {
-        this.currentMinStrValue = value;
-        this.currentMinStrValueInBytes = valueBytes;
-      } else if (compare(currentMaxStrValueInBytes, valueBytes) < 0) {
-        /*
-        Snowflake stores the first MAX_LOB_LEN characters of a string.
-        When truncating the max value, we increment the last max value
-        byte by one to ensure the max value stat is greater than the actual max value.
-         */
-        if (inputValue.length() > MAX_LOB_LEN) {
-          byte[] incrementedValueBytes = valueBytes.clone();
-          byte[] incrementedCollatedValueBytes = collatedValueBytes.clone();
-          incrementedValueBytes[MAX_LOB_LEN - 1]++;
-          incrementedCollatedValueBytes[MAX_LOB_LEN - 1]++;
-          String incrementedValue = new String(incrementedValueBytes);
-          this.currentMaxStrValue = incrementedValue;
-          this.currentMaxStrValueInBytes = incrementedValueBytes;
-        } else {
-          this.currentMaxStrValue = value;
-          this.currentMaxStrValueInBytes = valueBytes;
-        }
-      }
-
       // Collated comparison
       if (compare(currentMinColStrValueInBytes, collatedValueBytes) > 0) {
         this.currentMinColStrValue = value;
@@ -492,14 +450,6 @@ void addStrValue(String inputValue) {
     }
   }
 
-  String getCurrentMinStrValue() {
-    return currentMinStrValue;
-  }
-
-  String getCurrentMaxStrValue() {
-    return currentMaxStrValue;
-  }
-
   String getCurrentMinColStrValue() {
     return currentMinColStrValue;
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelDataTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelDataTest.java
index 69af1e8c6..cc072e070 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelDataTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelDataTest.java
@@ -107,13 +107,13 @@ public void testGetCombinedColumnStatsMap() {
     Assert.assertEquals(new BigInteger("10"), oneCombined.getCurrentMinIntValue());
     Assert.assertEquals(new BigInteger("17"), oneCombined.getCurrentMaxIntValue());
     Assert.assertEquals(-1, oneCombined.getDistinctValues());
-    Assert.assertNull(oneCombined.getCurrentMinStrValue());
-    Assert.assertNull(oneCombined.getCurrentMaxStrValue());
+    Assert.assertNull(oneCombined.getCurrentMinColStrValue());
+    Assert.assertNull(oneCombined.getCurrentMaxColStrValue());
     Assert.assertNull(oneCombined.getCurrentMinRealValue());
     Assert.assertNull(oneCombined.getCurrentMaxRealValue());
 
-    Assert.assertEquals("10", twoCombined.getCurrentMinStrValue());
-    Assert.assertEquals("17", twoCombined.getCurrentMaxStrValue());
+    Assert.assertEquals("10", twoCombined.getCurrentMinColStrValue());
+    Assert.assertEquals("17", twoCombined.getCurrentMaxColStrValue());
     Assert.assertEquals(-1, twoCombined.getDistinctValues());
     Assert.assertNull(twoCombined.getCurrentMinIntValue());
     Assert.assertNull(twoCombined.getCurrentMaxIntValue());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferStatsTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferStatsTest.java
index 50e5cac09..fe4027fc0 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferStatsTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferStatsTest.java
@@ -78,8 +78,6 @@ public void testEmptyState() throws Exception {
     Assert.assertNull(stats.getCurrentMaxColStrValue());
     Assert.assertNull(stats.getCurrentMinIntValue());
     Assert.assertNull(stats.getCurrentMaxIntValue());
-    Assert.assertNull(stats.getCurrentMinStrValue());
-    Assert.assertNull(stats.getCurrentMaxStrValue());
 
     Assert.assertEquals(0, stats.getCurrentNullCount());
     Assert.assertEquals(-1, stats.getDistinctValues());
@@ -90,22 +88,16 @@ public void testMinMaxStrNonCol() throws Exception {
     RowBufferStats stats = new RowBufferStats();
 
     stats.addStrValue("bob");
-    Assert.assertEquals("bob", stats.getCurrentMinStrValue());
-    Assert.assertEquals("bob", stats.getCurrentMaxStrValue());
     Assert.assertEquals("bob", stats.getCurrentMinColStrValue());
     Assert.assertEquals("bob", stats.getCurrentMaxColStrValue());
     Assert.assertEquals(-1, stats.getDistinctValues());
 
     stats.addStrValue("charlie");
-    Assert.assertEquals("bob", stats.getCurrentMinStrValue());
-    Assert.assertEquals("charlie", stats.getCurrentMaxStrValue());
     Assert.assertEquals("bob", stats.getCurrentMinColStrValue());
     Assert.assertEquals("charlie", stats.getCurrentMaxColStrValue());
     Assert.assertEquals(-1, stats.getDistinctValues());
 
     stats.addStrValue("alice");
-    Assert.assertEquals("alice", stats.getCurrentMinStrValue());
-    Assert.assertEquals("charlie", stats.getCurrentMaxStrValue());
     Assert.assertEquals("alice", stats.getCurrentMinColStrValue());
     Assert.assertEquals("charlie", stats.getCurrentMaxColStrValue());
     Assert.assertEquals(-1, stats.getDistinctValues());
@@ -122,12 +114,12 @@ public void testMinMaxStrNonCol() throws Exception {
   public void testStrTruncation() throws Exception {
     RowBufferStats stats = new RowBufferStats();
     stats.addStrValue("abcde|abcde|abcde|abcde|abcde|abcde|");
-    Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ab", stats.getCurrentMinStrValue());
-    Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ac", stats.getCurrentMaxStrValue());
+    Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ab", stats.getCurrentMinColStrValue());
+    Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ac", stats.getCurrentMaxColStrValue());
 
     stats.addStrValue("zabcde|abcde|abcde|abcde|abcde|abcde|");
-    Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ab", stats.getCurrentMinStrValue());
-    Assert.assertEquals("zabcde|abcde|abcde|abcde|abcde|b", stats.getCurrentMaxStrValue());
+    Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ab", stats.getCurrentMinColStrValue());
+    Assert.assertEquals("zabcde|abcde|abcde|abcde|abcde|b", stats.getCurrentMaxColStrValue());
 
     RowBufferStats ai = new RowBufferStats("en-ai");
     ai.addStrValue("abcde|abcde|abcde|abcde|abcde|abcde|");
@@ -146,22 +138,16 @@ public void testMinMaxStrCol() throws Exception {
     Assert.assertEquals("en-ci", stats.getCollationDefinitionString());
 
     stats.addStrValue("bob");
-    Assert.assertEquals("bob", stats.getCurrentMinStrValue());
-    Assert.assertEquals("bob", stats.getCurrentMaxStrValue());
     Assert.assertEquals("bob", stats.getCurrentMinColStrValue());
     Assert.assertEquals("bob", stats.getCurrentMaxColStrValue());
     Assert.assertEquals(-1, stats.getDistinctValues());
 
     stats.addStrValue("Bob");
-    Assert.assertEquals("Bob", stats.getCurrentMinStrValue());
-    Assert.assertEquals("bob", stats.getCurrentMaxStrValue());
     Assert.assertEquals("bob", stats.getCurrentMinColStrValue());
     Assert.assertEquals("bob", stats.getCurrentMaxColStrValue());
     Assert.assertEquals(-1, stats.getDistinctValues());
 
     stats.addStrValue("Alice");
-    Assert.assertEquals("Alice", stats.getCurrentMinStrValue());
-    Assert.assertEquals("bob", stats.getCurrentMaxStrValue());
     Assert.assertEquals("Alice", stats.getCurrentMinColStrValue());
     Assert.assertEquals("bob", stats.getCurrentMaxColStrValue());
     Assert.assertEquals(-1, stats.getDistinctValues());
@@ -194,8 +180,6 @@ public void testMinMaxInt() throws Exception {
 
     Assert.assertNull(stats.getCurrentMinRealValue());
     Assert.assertNull(stats.getCurrentMaxRealValue());
-    Assert.assertNull(stats.getCurrentMinStrValue());
-    Assert.assertNull(stats.getCurrentMaxStrValue());
     Assert.assertNull(stats.getCollationDefinitionString());
     Assert.assertNull(stats.getCurrentMinColStrValue());
     Assert.assertNull(stats.getCurrentMaxColStrValue());
@@ -223,8 +207,6 @@ public void testMinMaxReal() throws Exception {
 
     Assert.assertNull(stats.getCurrentMinIntValue());
     Assert.assertNull(stats.getCurrentMaxIntValue());
-    Assert.assertNull(stats.getCurrentMinStrValue());
-    Assert.assertNull(stats.getCurrentMaxStrValue());
     Assert.assertNull(stats.getCollationDefinitionString());
     Assert.assertNull(stats.getCurrentMinColStrValue());
     Assert.assertNull(stats.getCurrentMaxColStrValue());
@@ -277,8 +259,8 @@ public void testGetCombinedStats() throws Exception {
     Assert.assertEquals(-1, result.getDistinctValues());
     Assert.assertEquals(2, result.getCurrentNullCount());
 
-    Assert.assertNull(result.getCurrentMinStrValue());
-    Assert.assertNull(result.getCurrentMaxStrValue());
+    Assert.assertNull(result.getCurrentMinColStrValue());
+    Assert.assertNull(result.getCurrentMaxColStrValue());
     Assert.assertNull(result.getCurrentMinRealValue());
     Assert.assertNull(result.getCurrentMaxRealValue());
 
@@ -302,8 +284,6 @@ public void testGetCombinedStats() throws Exception {
     Assert.assertEquals(-1, result.getDistinctValues());
     Assert.assertEquals(0, result.getCurrentNullCount());
 
-    Assert.assertNull(result.getCurrentMinStrValue());
-    Assert.assertNull(result.getCurrentMaxStrValue());
     Assert.assertNull(result.getCollationDefinitionString());
     Assert.assertNull(result.getCurrentMinColStrValue());
     Assert.assertNull(result.getCurrentMaxColStrValue());
@@ -329,8 +309,6 @@ public void testGetCombinedStats() throws Exception {
     two.setCurrentMaxLength(1);
 
     result = RowBufferStats.getCombinedStats(one, two);
-    Assert.assertEquals("a", result.getCurrentMinStrValue());
-    Assert.assertEquals("g", result.getCurrentMaxStrValue());
     Assert.assertEquals("a", result.getCurrentMinColStrValue());
     Assert.assertEquals("g", result.getCurrentMaxColStrValue());
     Assert.assertEquals(-1, result.getDistinctValues());
@@ -359,8 +337,6 @@ public void testGetCombinedStats() throws Exception {
     two.incCurrentNullCount();
 
     result = RowBufferStats.getCombinedStats(one, two);
-    Assert.assertEquals("Alpha", result.getCurrentMinStrValue());
-    Assert.assertEquals("g", result.getCurrentMaxStrValue());
     Assert.assertEquals("a", result.getCurrentMinColStrValue());
     Assert.assertEquals("g", result.getCurrentMaxColStrValue());
     Assert.assertEquals(-1, result.getDistinctValues());
@@ -391,8 +367,8 @@ public void testGetCombinedStatsNull() throws Exception {
     Assert.assertEquals(-1, result.getDistinctValues());
     Assert.assertEquals(2, result.getCurrentNullCount());
 
-    Assert.assertNull(result.getCurrentMinStrValue());
-    Assert.assertNull(result.getCurrentMaxStrValue());
+    Assert.assertNull(result.getCurrentMinColStrValue());
+    Assert.assertNull(result.getCurrentMaxColStrValue());
     Assert.assertNull(result.getCurrentMinRealValue());
     Assert.assertNull(result.getCurrentMaxRealValue());
 
@@ -410,8 +386,8 @@ public void testGetCombinedStatsNull() throws Exception {
     Assert.assertEquals(-1, result.getDistinctValues());
     Assert.assertEquals(0, result.getCurrentNullCount());
 
-    Assert.assertNull(result.getCurrentMinStrValue());
-    Assert.assertNull(result.getCurrentMaxStrValue());
+    Assert.assertNull(result.getCurrentMinColStrValue());
+    Assert.assertNull(result.getCurrentMaxColStrValue());
     Assert.assertNull(result.getCurrentMinIntValue());
     Assert.assertNull(result.getCurrentMaxIntValue());
 
@@ -426,8 +402,8 @@ public void testGetCombinedStatsNull() throws Exception {
     one.incCurrentNullCount();
 
     result = RowBufferStats.getCombinedStats(one, two);
-    Assert.assertEquals("alpha", result.getCurrentMinStrValue());
-    Assert.assertEquals("g", result.getCurrentMaxStrValue());
+    Assert.assertEquals("alpha", result.getCurrentMinColStrValue());
+    Assert.assertEquals("g", result.getCurrentMaxColStrValue());
     Assert.assertEquals(-1, result.getDistinctValues());
     Assert.assertEquals(1, result.getCurrentNullCount());
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 6b22c8c85..73150f3fc 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -1179,8 +1179,8 @@ private void testStatsE2EHelper(ArrowRowBuffer rowBuffer) {
     Assert.assertEquals(0, columnEpStats.get("COLBIGINT").getCurrentNullCount());
     Assert.assertEquals(-1, columnEpStats.get("COLBIGINT").getDistinctValues());
 
-    Assert.assertEquals("alice", columnEpStats.get("COLCHAR").getCurrentMaxStrValue());
-    Assert.assertEquals("2", columnEpStats.get("COLCHAR").getCurrentMinStrValue());
+    Assert.assertEquals("alice", columnEpStats.get("COLCHAR").getCurrentMaxColStrValue());
+    Assert.assertEquals("2", columnEpStats.get("COLCHAR").getCurrentMinColStrValue());
     Assert.assertEquals(0, columnEpStats.get("COLCHAR").getCurrentNullCount());
     Assert.assertEquals(-1, columnEpStats.get("COLCHAR").getDistinctValues());
 
@@ -1572,9 +1572,9 @@ private void testE2EBinaryHelper(SnowflakeStreamingIngestChannelInternal channel
     Assert.assertEquals(3, result.getRowCount());
     Assert.assertEquals(11L, result.getColumnEps().get("COLBINARY").getCurrentMaxLength());
     Assert.assertEquals(
-        "Hello World", result.getColumnEps().get("COLBINARY").getCurrentMinStrValue());
+        "Hello World", result.getColumnEps().get("COLBINARY").getCurrentMinColStrValue());
     Assert.assertEquals(
-        "Honk Honk", result.getColumnEps().get("COLBINARY").getCurrentMaxStrValue());
+        "Honk Honk", result.getColumnEps().get("COLBINARY").getCurrentMaxColStrValue());
     Assert.assertEquals(1, result.getColumnEps().get("COLBINARY").getCurrentNullCount());
   }
 

From fefc8e249f3236bc20ce6492ffa60ee538f3bc89 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 15 Jun 2022 16:56:54 -0700
Subject: [PATCH 038/356] new release

---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index d92c06934..248576323 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
     <!-- Arifact name and version information -->
     <groupId>net.snowflake</groupId>
     <artifactId>snowflake-ingest-sdk</artifactId>
-    <version>1.0.2-beta.1</version>
+    <version>1.0.2-beta.2</version>
     <packaging>jar</packaging>
     <name>Snowflake Ingest SDK</name>
     <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index c71312db3..f63029431 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -119,7 +119,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.0.2-beta.1";
+  public static final String DEFAULT_VERSION = "1.0.2-beta.2";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From b9d17ea85e7e420bf517e0188380824815af16b5 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 16 Jun 2022 11:59:37 -0700
Subject: [PATCH 039/356] NO-SNOW: improve README and Snowpipe Streaming
 Example (#189)

Improve README and Snowpipe Streaming Example, it should be more clear for customers now
---
 README.md                                     | 54 +++++++++++--------
 .../SnowflakeStreamingIngestExample.java      | 54 ++++++++++---------
 2 files changed, 61 insertions(+), 47 deletions(-)

diff --git a/README.md b/README.md
index 78c4c88ed..101b9ea77 100644
--- a/README.md
+++ b/README.md
@@ -3,12 +3,13 @@ Snowflake Ingest Service Java SDK
 
 [![image](http://img.shields.io/:license-Apache%202-brightgreen.svg)](http://www.apache.org/licenses/LICENSE-2.0.txt)
 [![image](https://github.com/snowflakedb/snowflake-ingest-java/workflows/Snowpipe%20Java%20SDK%20Tests/badge.svg)](https://github.com/snowflakedb/snowflake-ingest-java/actions)
-[![image](https://codecov.io/gh/snowflakedb/snowflake-ingest-java/branch/master/graph/badge.svg)](https://codecov.io/gh/snowflakedb/snowflake-ingest-java)
 [![image](https://maven-badges.herokuapp.com/maven-central/net.snowflake/snowflake-ingest-sdk/badge.svg?style=plastic)](https://repo.maven.apache.org/maven2/net/snowflake/snowflake-ingest-sdk/)
 
 The Snowflake Ingest Service SDK allows users to ingest files into their
 Snowflake data warehouse in a programmatic fashion via key-pair
-authentication.
+authentication. Currently, we support ingestion through the following APIs:
+1. [Snowpipe](https://docs.snowflake.com/en/user-guide/data-load-snowpipe-rest-gs.html#client-requirement-java-or-python-sdk)
+2. [Snowpipe Streaming](https://docs.snowflake.com/en/LIMITEDACCESS/snowpipe-streaming.html) - Under Private Preview
 
 # Prerequisites
 
@@ -23,12 +24,12 @@ time.
 Snowflake Authentication for the Ingest Service requires creating a 2048
 bit RSA key pair and, registering the public key with Snowflake. For
 detailed instructions, please visit the relevant [Snowflake
-Documentation Page](docs.snowflake.net).
+Documentation Page](https://docs.snowflake.com/en/user-guide/key-pair-auth.html).
 
 ## Maven (Developers only)
 
-This SDK is developed as a [Maven](maven.apache.org) project. As a
-result, you'll need to install maven to build the projects and, run
+This SDK is developed as a Maven project. As a
+result, you'll need to install Maven to build the projects and, run
 tests.
 
 # Adding as a Dependency
@@ -52,6 +53,30 @@ dependencies {
 }
 ```
 
+# Example
+
+## Snowpipe
+
+Check out `SnowflakeIngestBasicExample.java`
+
+## Snowpipe Streaming
+
+Check out `SnowflakeStreamingIngestExample.java`, which performs following operations:
+1. Reads a JSON file which contains details regarding Snowflake Account, User, Role and Private Key. Take a look at `profile_streaming.json.example` for more details.
+    1. [Here](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication) are the steps required to generate a private key.
+2. Creates a `SnowflakeStreamingIngestClient` which can be used to open one or more Streaming Channels pointing to the same or different tables.
+3. Creates a `SnowflakeStreamingIngestChannel` against a Database, Schema and Table.
+    1. Please note: The database, schema and table is expected to be present before opening the Channel. Example SQL queries to create them:
+```sql
+create or replace database MY_DATABASE;
+create or replace schema MY_SCHEMA;
+create or replace table MY_TABLE(c1 number);
+```
+4. Inserts 1000 rows into the channel created in 3rd step using the `insertRows` API on the Channel object
+    1. `insertRows` API also takes in an optional `offsetToken` String which can be associated to this batch of rows.
+5. Calls `getLatestCommittedOffsetToken` on the channel until the appropriate offset is found in Snowflake.
+6. Close the channel when the ingestion is done to make sure everything is committed.
+
 # Building From Source
 
 If you would like to build this project from source you can run the
@@ -100,21 +125,4 @@ you would need to remove the following scope limits in pom.xml
 We use [Google Java format](https://github.com/google/google-java-format) to format the code. To format all files, run:
 ```bash
 ./format.sh
-````
-
-# Snowpipe Streaming Example (Still in Preview)
-
-Run File `SnowflakeStreamingIngestExample.java` which performs following operations.
-1. Reads a JSON file which contains details regarding Snowflake Account, User, Role and Private Key. Take a look at `profile_streaming.json.example` for more details.
-   1. [Here](https://docs.snowflake.com/en/user-guide/key-pair-auth.html#configuring-key-pair-authentication) are the steps required to generate a private key.
-2. Creates a `SnowflakeStreamingIngestClient` which can be used to open one or more Streaming Channels against a table.
-3. Creates a `SnowflakeStreamingIngestChannel` against a Database, Schema and Table name.
-   1. Please note: A Table is expected to be present before opening a Channel. Use following SQL queries and place respective Database, Schema and Table names in `profile_streaming.json` file
-```sql
-create or replace database MY_DATABASE;
-create or replace schema MY_SCHEMA;
-create or replace table MY_TABLE(c1 number);
-```
-4. Inserts a few rows (1000) into a channel created in 3rd step using the `insertRows` API on the Channel object
-   1. `insertRows` API also takes in an optional `offsetToken` String which can be associated to this batch of rows. 
-5. Calls `getLatestCommittedOffsetToken` on the channel until the appropriate offset is found in Snowflake.
\ No newline at end of file
+````
\ No newline at end of file
diff --git a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
index c1d065241..2b12aba9f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
+++ b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
@@ -24,7 +24,10 @@
  * <p>Please read the README.md file for detailed steps
  */
 public class SnowflakeStreamingIngestExample {
-  private static String PROFILE_PATH = "profile_streaming.json";
+  // Please follow the example in profile_streaming.json.example to see the required properties, or
+  // if you have already set up profile.json with Snowpipe before, all you need is to add the "role"
+  // property.
+  private static String PROFILE_PATH = "profile.json";
   private static final ObjectMapper mapper = new ObjectMapper();
 
   public static void main(String[] args) throws Exception {
@@ -38,22 +41,24 @@ public static void main(String[] args) throws Exception {
 
     // Create a streaming ingest client
     try (SnowflakeStreamingIngestClient client =
-        SnowflakeStreamingIngestClientFactory.builder("CLIENT").setProperties(props).build()) {
+        SnowflakeStreamingIngestClientFactory.builder("MY_CLIENT").setProperties(props).build()) {
 
-      // Create an open channel request on table T_STREAMINGINGEST
+      // Create an open channel request on table MY_TABLE, note that the corresponding
+      // db/schema/table needs to be present
+      // Example: create or replace table MY_TABLE(c1 number);
       OpenChannelRequest request1 =
           OpenChannelRequest.builder("MY_CHANNEL")
               .setDBName("MY_DATABASE")
               .setSchemaName("MY_SCHEMA")
               .setTableName("MY_TABLE")
-              .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+              .setOnErrorOption(
+                  OpenChannelRequest.OnErrorOption.CONTINUE) // Another ON_ERROR option is ABORT
               .build();
 
       // Open a streaming ingest channel from the given client
       SnowflakeStreamingIngestChannel channel1 = client.openChannel(request1);
 
-      // Insert few rows into the channel (Using insertRows API), along with the offset Token which
-      // corresponds to the row number
+      // Insert rows into the channel (Using insertRows API)
       final int totalRowsInTable = 1000;
       for (int val = 0; val < totalRowsInTable; val++) {
         Map<String, Object> row = new HashMap<>();
@@ -61,33 +66,34 @@ public static void main(String[] args) throws Exception {
         // c1 corresponds to the column name in table
         row.put("c1", val);
 
+        // Insert the row with the current offset_token
         InsertValidationResponse response = channel1.insertRow(row, String.valueOf(val));
         if (response.hasErrors()) {
-          // Simply throw if there is an exception
+          // Simply throw if there is an exception, or you can do whatever you want with the
+          // erroneous row
           throw response.getInsertErrors().get(0).getException();
         }
       }
 
-      // Polling Snowflake to fetch offset token registered in Snowflake
-      final int expectedOffsetTokenInSnowflake = 999; // because it goes from 0 to 999
-      String offsetTokenFromSnowflake = channel1.getLatestCommittedOffsetToken();
-
+      // If needed, you can check the offset_token registered in Snowflake to make sure everything
+      // is committed
+      final int expectedOffsetTokenInSnowflake = totalRowsInTable - 1; // 0 based offset_token
       final int maxRetries = 10;
       int retryCount = 0;
-      while (offsetTokenFromSnowflake == null
-          || !offsetTokenFromSnowflake.equals(String.valueOf(expectedOffsetTokenInSnowflake))) {
-        Thread.sleep(1_000);
-        offsetTokenFromSnowflake = channel1.getLatestCommittedOffsetToken();
-        retryCount++;
-        if (retryCount >= maxRetries) {
-          System.out.println(
-              String.format(
-                  "Failed to look for required OffsetToken in Snowflake:%s after MaxRetryCounts:%s",
-                  expectedOffsetTokenInSnowflake, maxRetries));
-          System.exit(1);
+
+      do {
+        String offsetTokenFromSnowflake = channel1.getLatestCommittedOffsetToken();
+        if (offsetTokenFromSnowflake != null
+            && offsetTokenFromSnowflake.equals(String.valueOf(expectedOffsetTokenInSnowflake))) {
+          System.out.println("SUCCESSFULLY inserted " + totalRowsInTable + " rows");
+          break;
         }
-      }
-      System.out.println("SUCCESSFULLY inserted " + totalRowsInTable + " rows");
+        retryCount++;
+      } while (retryCount < maxRetries);
+
+      // Close the channel, the function internally will make sure everything is committed (or throw
+      // an exception if there is any issue)
+      channel1.close().get();
     }
   }
 }

From 488b36c68ea166741ee5b230e25db2885a523aed Mon Sep 17 00:00:00 2001
From: sfc-gh-jfan <jiazhen.fan@snowflake.com>
Date: Fri, 17 Jun 2022 09:45:10 -0700
Subject: [PATCH 040/356] minor update

---
 .github/workflows/snyk-issue.yml | 2 +-
 .github/workflows/snyk-pr.yml    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/snyk-issue.yml b/.github/workflows/snyk-issue.yml
index 7131bd14b..74d6e1aaf 100644
--- a/.github/workflows/snyk-issue.yml
+++ b/.github/workflows/snyk-issue.yml
@@ -23,7 +23,7 @@ jobs:
       with:
         snyk_org: ${{ secrets.SNYK_ORG_ID_PUBLIC_REPO }}
         snyk_token:  ${{ secrets.SNYK_GITHUB_INTEGRATION_TOKEN }}
-        jira_token: ${{ secrets.JIRA_TOKEN }}
+        jira_token: ${{ secrets.JIRA_TOKEN_PUBLIC_REPO }}
       env:
         GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
diff --git a/.github/workflows/snyk-pr.yml b/.github/workflows/snyk-pr.yml
index 9a3c117f1..048b86917 100644
--- a/.github/workflows/snyk-pr.yml
+++ b/.github/workflows/snyk-pr.yml
@@ -26,6 +26,6 @@ jobs:
       env:
         PR_TITLE: ${{ github.event.pull_request.title }}
       with:
-        jira_token: ${{ secrets.JIRA_TOKEN }}
+        jira_token: ${{ secrets.JIRA_TOKEN_PUBLIC_REPO }}
         gh_token: ${{ secrets.GITHUB_TOKEN }}
         amend: false # true if you want the commit to be amended with the JIRA number

From 32d406a8814fe8e9237af88e2bb7aca06984728b Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 30 Jun 2022 21:33:06 -0700
Subject: [PATCH 041/356] SNOW-619777, SNOW-606454: fix default min/max value
 in EP for real column (#194)

When all the values in a REAL column are NULLs, the server side expects the min/max in EP to be 0.0, but the client SDK is sending NULL to the server side and causing exception. This PR fixes this issue and it also contains a few small related changes.
---
 .../streaming/internal/ArrowRowBuffer.java    |  2 +
 .../internal/FileColumnProperties.java        | 13 ++++-
 .../streaming/internal/RowBufferStats.java    | 19 +++----
 ...nowflakeStreamingIngestClientInternal.java | 17 +++++-
 .../streaming/internal/RowBufferTest.java     | 19 +++++--
 .../SnowflakeStreamingIngestClientTest.java   |  5 +-
 .../streaming/internal/StreamingIngestIT.java | 56 +++++++++++++++++++
 7 files changed, 110 insertions(+), 21 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 7b8074078..46faa1fb2 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -670,6 +670,7 @@ private float convertRowToArrow(
     // Verify all the input columns are valid
     Set<String> inputColumnNames = verifyInputColumns(row);
 
+    // Insert values to the corresponding arrow buffers
     float rowBufferSize = 0F;
     for (Map.Entry<String, Object> entry : row.entrySet()) {
       rowBufferSize += 0.125; // 1/8 for null value bitmap
@@ -956,6 +957,7 @@ private float convertRowToArrow(
 
     // Insert nulls to the columns that doesn't show up in the input
     for (String columnName : Sets.difference(this.fields.keySet(), inputColumnNames)) {
+      rowBufferSize += 0.125; // 1/8 for null value bitmap
       insertNull(
           sourceVectors.getVector(this.fields.get(columnName)),
           statsMap.get(columnName),
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java b/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java
index ecc6dc031..581d96bbd 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java
@@ -38,6 +38,9 @@ class FileColumnProperties {
   // Default value to use for min/max int when all data in the given column is NULL
   public static final BigInteger DEFAULT_MIN_MAX_INT_VAL_FOR_EP = BigInteger.valueOf(0);
 
+  // Default value to use for min/max real when all data in the given column is NULL
+  public static final Double DEFAULT_MIN_MAX_REAL_VAL_FOR_EP = 0d;
+
   FileColumnProperties(RowBufferStats stats) {
     this.setCollation(stats.getCollationDefinitionString());
     this.setMaxIntValue(
@@ -48,8 +51,14 @@ class FileColumnProperties {
         stats.getCurrentMinIntValue() == null
             ? DEFAULT_MIN_MAX_INT_VAL_FOR_EP
             : stats.getCurrentMinIntValue());
-    this.setMinRealValue(stats.getCurrentMinRealValue());
-    this.setMaxRealValue(stats.getCurrentMaxRealValue());
+    this.setMinRealValue(
+        stats.getCurrentMinRealValue() == null
+            ? DEFAULT_MIN_MAX_REAL_VAL_FOR_EP
+            : stats.getCurrentMinRealValue());
+    this.setMaxRealValue(
+        stats.getCurrentMaxRealValue() == null
+            ? DEFAULT_MIN_MAX_REAL_VAL_FOR_EP
+            : stats.getCurrentMaxRealValue());
     this.setMaxLength(stats.getCurrentMaxLength());
 
     // Collated and non-collated strings are intentionally equal here as required by Snowflake
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
index 6aaad8e6f..54395b35c 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
@@ -11,6 +11,7 @@
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
+import java.util.Objects;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 
@@ -252,11 +253,7 @@ byte[] performConversion(String input) {
 
       if (icuCollationString == null) {
         // No need for ICU key generation, convert (modified) input to UTF-8 bytes
-        try {
-          return input.getBytes("UTF-8");
-        } catch (java.io.UnsupportedEncodingException e) {
-          throw new SFException(ErrorCode.INTERNAL_ERROR, "Failed encoding string to UTF-8");
-        }
+        return input.getBytes(StandardCharsets.UTF_8);
       }
 
       // ICU collation is needed
@@ -347,7 +344,7 @@ byte[] getCollatedBytes(String value) {
 
   // TODO performance test this vs in place update
   static RowBufferStats getCombinedStats(RowBufferStats left, RowBufferStats right) {
-    if (left.getCollationDefinitionString() != right.collationDefinitionString) {
+    if (!Objects.equals(left.getCollationDefinitionString(), right.collationDefinitionString)) {
       throw new SFException(
           ErrorCode.INVALID_COLLATION_STRING,
           "Tried to combine stats for different collations",
@@ -398,8 +395,8 @@ void addStrValue(String inputValue) {
     String value =
         inputValue.length() > MAX_LOB_LEN ? inputValue.substring(0, MAX_LOB_LEN) : inputValue;
 
-    byte[] valueBytes = value != null ? value.getBytes(StandardCharsets.UTF_8) : null;
-    byte[] collatedValueBytes = value != null ? getCollatedBytes(value) : null;
+    byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
+    byte[] collatedValueBytes = getCollatedBytes(value);
 
     // Check if new min/max string
     if (this.currentMinColStrValue == null) {
@@ -416,8 +413,7 @@ void addStrValue(String inputValue) {
         byte[] incrementedCollatedValueBytes = collatedValueBytes.clone();
         incrementedValueBytes[MAX_LOB_LEN - 1]++;
         incrementedCollatedValueBytes[MAX_LOB_LEN - 1]++;
-        String incrementedValue = new String(incrementedValueBytes);
-        this.currentMaxColStrValue = incrementedValue;
+        this.currentMaxColStrValue = new String(incrementedValueBytes);
         this.currentMaxColStrValueInBytes = incrementedCollatedValueBytes;
       } else {
         this.currentMaxColStrValue = value;
@@ -439,8 +435,7 @@ void addStrValue(String inputValue) {
           byte[] incrementedCollatedValueBytes = collatedValueBytes.clone();
           incrementedValueBytes[MAX_LOB_LEN - 1]++;
           incrementedCollatedValueBytes[MAX_LOB_LEN - 1]++;
-          String incrementedValue = new String(incrementedValueBytes);
-          this.currentMaxColStrValue = incrementedValue;
+          this.currentMaxColStrValue = new String(incrementedValueBytes);
           this.currentMaxColStrValueInBytes = incrementedCollatedValueBytes;
         } else {
           this.currentMaxColStrValue = value;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index f6d97ee2d..2787c9088 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -351,6 +351,19 @@ ChannelsStatusResponse getChannelsStatus(List<SnowflakeStreamingIngestChannelInt
         throw new SFException(ErrorCode.CHANNEL_STATUS_FAILURE, response.getMessage());
       }
 
+      for (int idx = 0; idx < channels.size(); idx++) {
+        SnowflakeStreamingIngestChannelInternal channel = channels.get(idx);
+        ChannelsStatusResponse.ChannelStatusResponseDTO channelStatus =
+            response.getChannels().get(idx);
+        if (channelStatus.getStatusCode() != RESPONSE_SUCCESS) {
+          logger.logWarn(
+              "Channel has failure status_code, name={}, channel_sequencer={}," + " status_code={}",
+              channel.getFullyQualifiedName(),
+              channel.getChannelSequencer(),
+              channelStatus.getStatusCode());
+        }
+      }
+
       return response;
     } catch (IOException | IngestResponseException e) {
       throw new SFException(e, ErrorCode.CHANNEL_STATUS_FAILURE, e.getMessage());
@@ -443,8 +456,8 @@ void registerBlobs(List<BlobMetadata> blobs, final int executionCount) {
                                         } else {
                                           logger.logWarn(
                                               "Channel has been invalidated because of failure"
-                                                  + " response, name={}, channel sequencer={},"
-                                                  + " status code={}, executionCount={}",
+                                                  + " response, name={}, channel_sequencer={},"
+                                                  + " status_code={}, executionCount={}",
                                               channelStatus.getChannelName(),
                                               channelStatus.getChannelSequencer(),
                                               channelStatus.getStatusCode(),
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 73150f3fc..84e39a8ed 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -837,19 +837,21 @@ public void testBuildEpInfoFromStats() {
 
   @Test
   public void testBuildEpInfoFromNullColumnStats() {
-    final String colName = "intCol";
+    final String intColName = "intCol";
+    final String realColName = "realCol";
     Map<String, RowBufferStats> colStats = new HashMap<>();
 
     RowBufferStats stats = new RowBufferStats();
     stats.incCurrentNullCount();
 
-    colStats.put(colName, stats);
+    colStats.put(intColName, stats);
+    colStats.put(realColName, stats);
 
     EpInfo result = ArrowRowBuffer.buildEpInfoFromStats(2, colStats);
     Map<String, FileColumnProperties> columnResults = result.getColumnEps();
-    Assert.assertEquals(1, columnResults.keySet().size());
+    Assert.assertEquals(2, columnResults.keySet().size());
 
-    FileColumnProperties intColumnResult = columnResults.get(colName);
+    FileColumnProperties intColumnResult = columnResults.get(intColName);
     Assert.assertEquals(-1, intColumnResult.getDistinctValues());
     Assert.assertEquals(
         FileColumnProperties.DEFAULT_MIN_MAX_INT_VAL_FOR_EP, intColumnResult.getMinIntValue());
@@ -857,6 +859,15 @@ public void testBuildEpInfoFromNullColumnStats() {
         FileColumnProperties.DEFAULT_MIN_MAX_INT_VAL_FOR_EP, intColumnResult.getMaxIntValue());
     Assert.assertEquals(1, intColumnResult.getNullCount());
     Assert.assertEquals(0, intColumnResult.getMaxLength());
+
+    FileColumnProperties realColumnResult = columnResults.get(realColName);
+    Assert.assertEquals(-1, intColumnResult.getDistinctValues());
+    Assert.assertEquals(
+        FileColumnProperties.DEFAULT_MIN_MAX_REAL_VAL_FOR_EP, realColumnResult.getMinRealValue());
+    Assert.assertEquals(
+        FileColumnProperties.DEFAULT_MIN_MAX_REAL_VAL_FOR_EP, realColumnResult.getMaxRealValue());
+    Assert.assertEquals(1, realColumnResult.getNullCount());
+    Assert.assertEquals(0, realColumnResult.getMaxLength());
   }
 
   @Test
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index 3a27c0339..0b4f1b3a5 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -301,10 +301,13 @@ private String generateAESKey(PrivateKey key, char[] passwd)
 
   @Test
   public void testGetChannelsStatusWithRequest() throws Exception {
+    ChannelsStatusResponse.ChannelStatusResponseDTO channelStatus =
+        new ChannelsStatusResponse.ChannelStatusResponseDTO();
+    channelStatus.setStatusCode((long) RESPONSE_SUCCESS);
     ChannelsStatusResponse response = new ChannelsStatusResponse();
     response.setStatusCode(0L);
     response.setMessage("honk");
-    response.setChannels(new ArrayList<>());
+    response.setChannels(Collections.singletonList(channelStatus));
     String responseString = objectMapper.writeValueAsString(response);
 
     CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 5c16b7b78..3c0d7c4fb 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -861,6 +861,62 @@ public void testChannelClose() throws Exception {
     channel1.close().get();
   }
 
+  @Test
+  public void testNullValuesOnMultiDataTypes() throws Exception {
+    String nullValuesOnMultiDataTypesTable = "test_null_values_on_multi_data_types";
+    jdbcConnection
+        .createStatement()
+        .execute(
+            String.format(
+                "create or replace table %s (c0 int, c1 number, c2 decimal, c3 bigint, c4 float,"
+                    + " c5 real, c6 varchar, c7 char, c8 string, c9 text, c10 binary, c11 boolean,"
+                    + " c12 date, c13 time, c14 timestamp, c15 variant, c16 object, c17 array);",
+                nullValuesOnMultiDataTypesTable));
+
+    OpenChannelRequest request =
+        OpenChannelRequest.builder("CHANNEL")
+            .setDBName(TEST_DB)
+            .setSchemaName(TEST_SCHEMA)
+            .setTableName(nullValuesOnMultiDataTypesTable)
+            .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+            .build();
+
+    // Open a streaming ingest channel from the given client
+    SnowflakeStreamingIngestChannel channel1 = client.openChannel(request);
+    for (int val = 0; val < 1000; val++) {
+      Map<String, Object> row = new HashMap<>();
+      row.put("c0", val);
+      verifyInsertValidationResponse(channel1.insertRow(row, Integer.toString(val)));
+    }
+
+    // Sleep for a few seconds to make sure a flush has finished
+    Thread.sleep(5000);
+
+    // Insert one row with all NULLs to make sure it works
+    Map<String, Object> row = new HashMap<>();
+    for (int idx = 0; idx < 18; idx++) {
+      row.put("c" + idx, null);
+    }
+    verifyInsertValidationResponse(channel1.insertRow(row, "0"));
+
+    // Close the channel to make sure everything is committed
+    channel1.close().get();
+
+    // Query the table in the end to make sure everything is readable
+    ResultSet result =
+        jdbcConnection
+            .createStatement()
+            .executeQuery(
+                String.format(
+                    "select top 1 c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14,"
+                        + " c15, c16, c17 from %s.%s.%s",
+                    TEST_DB, TEST_SCHEMA, nullValuesOnMultiDataTypesTable));
+    result.next();
+    for (int idx = 1; idx < 18; idx++) {
+      Assert.assertNull(result.getObject(idx));
+    }
+  }
+
   /** Verify the insert validation response and throw the exception if needed */
   private void verifyInsertValidationResponse(InsertValidationResponse response) {
     if (response.hasErrors()) {

From 279df60f41cd3311fc5da832bb6568c9ef715011 Mon Sep 17 00:00:00 2001
From: Jay Patel <jay.patel@snowflake.com>
Date: Wed, 13 Jul 2022 16:55:14 -0700
Subject: [PATCH 042/356] NO-SNOW http connection pooling manager (#193)

* SNOW-585421 Add PoolingHttpClientConnectionManager as the connection manager with default maxConnections per route and maxTotalConns

- The default also uses PoolingHttpClientConnectionManager, but is
  limited by number of maxConnections and maxConnectionsPerRoute (20, 2)
  respectively.
- Also introducing a separate thread which periodically closes idle
  connections.

Log pool stats for PoolingHttpClientConnectionManager

Minor change, javadoc

Convert IdleConnectionMonitorThread to deamon thread

Resolve comments and shutdown thread

- Make the thread instance static.
- Add test to verify the behavior.

* Fix IllegalStateException found while testing

- When different threads enter initIdleConnectionMonitoringThread, it is
  possible that thread.start results into IllegalThreadStateException,
  hence adding a synchronization around this method by adding a static
  lock object

* Review comments
---
 .../snowflake/ingest/SimpleIngestManager.java |   1 +
 .../net/snowflake/ingest/utils/HttpUtil.java  | 159 +++++++++++++++++-
 .../net/snowflake/ingest/SimpleIngestIT.java  | 107 ++++++++----
 3 files changed, 228 insertions(+), 39 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
index 2ccfea05c..756b5df9c 100644
--- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
+++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
@@ -689,6 +689,7 @@ public ClientStatusResponse getClientStatus(UUID requestId)
   @Override
   public void close() {
     builder.closeResources();
+    HttpUtil.shutdownHttpConnectionManagerDaemonThread();
   }
 
   /* Used for testing */
diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index 565ffc408..ad1139974 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -8,7 +8,9 @@
 
 import java.security.Security;
 import java.util.Properties;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
 import javax.net.ssl.SSLContext;
 import net.snowflake.client.core.SFSessionProperty;
 import org.apache.http.HttpHost;
@@ -23,13 +25,15 @@
 import org.apache.http.client.ServiceUnavailableRetryStrategy;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.conn.routing.HttpRoute;
 import org.apache.http.conn.ssl.DefaultHostnameVerifier;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.conn.DefaultProxyRoutePlanner;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.pool.PoolStats;
 import org.apache.http.protocol.HttpContext;
 import org.apache.http.ssl.SSLContexts;
 import org.slf4j.Logger;
@@ -46,15 +50,41 @@ public class HttpUtil {
 
   private static final String PROXY_SCHEME = "http";
   private static final int MAX_RETRIES = 3;
-  static final int DEFAULT_CONNECTION_TIMEOUT = 1; // minute
-  static final int DEFAULT_HTTP_CLIENT_SOCKET_TIMEOUT = 5; // minutes
   private static CloseableHttpClient httpClient;
 
+  private static PoolingHttpClientConnectionManager connectionManager;
+
+  private static IdleConnectionMonitorThread idleConnectionMonitorThread;
+
+  /**
+   * This lock is to synchronize on idleConnectionMonitorThread to avoid setting starting a thread
+   * which was already started. (To avoid {@link IllegalThreadStateException})
+   */
+  private static ReentrantLock idleConnectionMonitorThreadLock = new ReentrantLock(true);
+
+  private static final int DEFAULT_CONNECTION_TIMEOUT_MINUTES = 1;
+  private static final int DEFAULT_HTTP_CLIENT_SOCKET_TIMEOUT_MINUTES = 5;
+
+  // Default is 2, but scaling it up to 100 to match with default_max_connections
+  private static final int DEFAULT_MAX_CONNECTIONS_PER_ROUTE = 100;
+
+  // 100 is close to max partition number we have seen for a kafka topic ingesting into snowflake.
+  private static final int DEFAULT_MAX_CONNECTIONS = 100;
+
+  // Interval in which we check if there are connections which needs to be closed.
+  private static final long IDLE_HTTP_CONNECTION_MONITOR_THREAD_INTERVAL_MS =
+      TimeUnit.SECONDS.toMillis(5);
+
+  // Only connections that are currently owned, not checked out, are subject to idle timeouts.
+  private static final int DEFAULT_IDLE_CONNECTION_TIMEOUT_SECONDS = 30;
+
   public static CloseableHttpClient getHttpClient() {
     if (httpClient == null) {
       initHttpClient();
     }
 
+    initIdleConnectionMonitoringThread();
+
     return httpClient;
   }
 
@@ -76,20 +106,32 @@ private static void initHttpClient() {
     RequestConfig requestConfig =
         RequestConfig.custom()
             .setConnectTimeout(
-                (int) TimeUnit.MILLISECONDS.convert(DEFAULT_CONNECTION_TIMEOUT, TimeUnit.MINUTES))
+                (int)
+                    TimeUnit.MILLISECONDS.convert(
+                        DEFAULT_CONNECTION_TIMEOUT_MINUTES, TimeUnit.MINUTES))
             .setConnectionRequestTimeout(
-                (int) TimeUnit.MILLISECONDS.convert(DEFAULT_CONNECTION_TIMEOUT, TimeUnit.MINUTES))
+                (int)
+                    TimeUnit.MILLISECONDS.convert(
+                        DEFAULT_CONNECTION_TIMEOUT_MINUTES, TimeUnit.MINUTES))
             .setSocketTimeout(
                 (int)
                     TimeUnit.MILLISECONDS.convert(
-                        DEFAULT_HTTP_CLIENT_SOCKET_TIMEOUT, TimeUnit.MINUTES))
+                        DEFAULT_HTTP_CLIENT_SOCKET_TIMEOUT_MINUTES, TimeUnit.MINUTES))
             .build();
+
+    // Below pooling client connection manager uses time_to_live value as -1 which means it will not
+    // refresh a persisted connection
+    connectionManager = new PoolingHttpClientConnectionManager();
+    connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_CONNECTIONS_PER_ROUTE);
+    connectionManager.setMaxTotal(DEFAULT_MAX_CONNECTIONS);
+
     /**
      * Use a anonymous class to implement the interface ServiceUnavailableRetryStrategy() The max
      * retry time is 3. The interval time is backoff.
      */
     HttpClientBuilder clientBuilder =
-        HttpClients.custom()
+        HttpClientBuilder.create()
+            .setConnectionManager(connectionManager)
             .setSSLSocketFactory(f)
             .setServiceUnavailableRetryStrategy(getServiceUnavailableRetryStrategy())
             .setRetryHandler(getHttpRequestRetryHandler())
@@ -126,6 +168,27 @@ private static void initHttpClient() {
     httpClient = clientBuilder.build();
   }
 
+  /** Starts a daemon thread to monitor idle connections in http connection manager. */
+  private static void initIdleConnectionMonitoringThread() {
+    idleConnectionMonitorThreadLock.lock();
+    try {
+      // Start a new thread only if connectionManager was init before and
+      // daemon thread was not started before or was closed because of SimpleIngestManager close
+      if (connectionManager != null
+          && (idleConnectionMonitorThread == null || idleConnectionMonitorThread.isShutdown())) {
+        // Monitors in a separate thread where it closes any idle connections
+        // https://hc.apache.org/httpcomponents-client-4.5.x/current/tutorial/html/connmgmt.html
+        idleConnectionMonitorThread = new IdleConnectionMonitorThread(connectionManager);
+        idleConnectionMonitorThread.setDaemon(true);
+        idleConnectionMonitorThread.start();
+      }
+    } catch (Exception e) {
+      LOGGER.warn("Unable to start Daemon thread for Http Idle Connection Monitoring", e);
+    } finally {
+      idleConnectionMonitorThreadLock.unlock();
+    }
+  }
+
   private static ServiceUnavailableRetryStrategy getServiceUnavailableRetryStrategy() {
     return new ServiceUnavailableRetryStrategy() {
       private int executionCount = 0;
@@ -235,4 +298,86 @@ public static Properties generateProxyPropertiesForJDBC() {
     }
     return proxyProperties;
   }
+
+  /** Thread to monitor expired and idle connection, if found clear it and return it back to pool */
+  private static class IdleConnectionMonitorThread extends Thread {
+
+    private final PoolingHttpClientConnectionManager connectionManager;
+    private volatile boolean shutdown;
+
+    public IdleConnectionMonitorThread(PoolingHttpClientConnectionManager connectionManager) {
+      super();
+      this.connectionManager = connectionManager;
+    }
+
+    @Override
+    public void run() {
+      try {
+        LOGGER.debug("Starting Idle Connection Monitor Thread ");
+        synchronized (this) {
+          while (!shutdown) {
+            wait(IDLE_HTTP_CONNECTION_MONITOR_THREAD_INTERVAL_MS);
+
+            StringBuilder sb = new StringBuilder();
+
+            sb.append(
+                createPoolStatsInfo("Total Pool Stats = ", connectionManager.getTotalStats()));
+            Set<HttpRoute> routes = connectionManager.getRoutes();
+
+            if (routes != null) {
+              for (HttpRoute route : routes) {
+                sb.append(createPoolStatsForRoute(connectionManager, route));
+              }
+            }
+
+            LOGGER.debug("[IdleConnectionMonitorThread] Pool Stats:\n" + sb);
+
+            // Close expired connections
+            connectionManager.closeExpiredConnections();
+            // Optionally, close connections
+            // that have been idle longer than 30 sec
+            connectionManager.closeIdleConnections(
+                DEFAULT_IDLE_CONNECTION_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+          }
+        }
+      } catch (InterruptedException ex) {
+        LOGGER.warn("Terminating Idle Connection Monitor Thread ");
+      }
+    }
+
+    private void shutdown() {
+      if (!shutdown) {
+        LOGGER.debug("Shutdown Idle Connection Monitor Thread ");
+        shutdown = true;
+        synchronized (this) {
+          notifyAll();
+        }
+      }
+    }
+
+    private boolean isShutdown() {
+      return this.shutdown;
+    }
+  }
+
+  /** Shuts down the daemon thread. */
+  public static void shutdownHttpConnectionManagerDaemonThread() {
+    idleConnectionMonitorThread.shutdown();
+  }
+
+  /** Create Pool stats for a route */
+  private static String createPoolStatsForRoute(
+      PoolingHttpClientConnectionManager connectionManager, HttpRoute route) {
+    PoolStats routeStats = connectionManager.getStats(route);
+    return createPoolStatsInfo(
+        String.format("Pool Stats for route %s = ", route.getTargetHost().toURI()), routeStats);
+  }
+
+  /** Returns a string with a title and pool stats. */
+  private static String createPoolStatsInfo(String title, PoolStats poolStats) {
+    if (poolStats != null) {
+      return title + poolStats + "\n";
+    }
+    return title;
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
index cdcb0e52b..207db28a7 100644
--- a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
@@ -39,6 +39,7 @@
 public class SimpleIngestIT {
   private final String TEST_FILE_NAME = "test1.csv";
   private final String TEST_FILE_NAME_2 = "test2.csv";
+  private final String PIPE_NAME_PREFIX = "ingest_sdk_test_pipe_";
 
   private String testFilePath = null;
   private String testFilePath_2 = null;
@@ -54,6 +55,10 @@ public class SimpleIngestIT {
   // the object mapper we use for deserialization
   static ObjectMapper mapper = new ObjectMapper();
 
+  private final Random RAND = new Random();
+
+  private final Long RAND_NUM = Math.abs(RAND.nextLong());
+
   /** Create test table and pipe */
   @Before
   public void beforeAll() throws Exception {
@@ -66,19 +71,16 @@ public void beforeAll() throws Exception {
     testFilePath_2 = resource.getFile();
 
     // create stage, pipe, and table
-    Random rand = new Random();
 
-    Long num = Math.abs(rand.nextLong());
+    tableName = "ingest_sdk_test_table_" + RAND_NUM;
 
-    tableName = "ingest_sdk_test_table_" + num;
+    pipeName = PIPE_NAME_PREFIX + RAND_NUM;
 
-    pipeName = "ingest_sdk_test_pipe_" + num;
+    pipeWithPatternName = "ingest_sdk_test_pipe_pattern_" + RAND_NUM;
 
-    pipeWithPatternName = "ingest_sdk_test_pipe_pattern_" + num;
+    stageName = "ingest_sdk_test_stage_" + RAND_NUM;
 
-    stageName = "ingest_sdk_test_stage_" + num;
-
-    stageWithPatternName = "ingest_sdk_test_stage_pattern" + num;
+    stageWithPatternName = "ingest_sdk_test_stage_pattern" + RAND_NUM;
 
     TestUtils.executeQuery("create or replace table " + tableName + " (str string, num int)");
 
@@ -86,13 +88,7 @@ public void beforeAll() throws Exception {
 
     TestUtils.executeQuery("create or replace stage " + stageWithPatternName);
 
-    TestUtils.executeQuery(
-        "create or replace pipe "
-            + pipeName
-            + " as copy into "
-            + tableName
-            + " from @"
-            + stageName);
+    createPipe(tableName, stageName, pipeName);
 
     TestUtils.executeQuery(
         "create or replace pipe "
@@ -127,23 +123,7 @@ public void testSimpleIngest() throws Exception {
     // create ingest manager
     SimpleIngestManager manager = TestUtils.getManager(pipeName);
 
-    // create a file wrapper
-    StagedFileWrapper myFile = new StagedFileWrapper(TEST_FILE_NAME, null);
-
-    // get an insert response after we submit
-    IngestResponse insertResponse = manager.ingestFile(myFile, null);
-
-    assertEquals("SUCCESS", insertResponse.getResponseCode());
-
-    // Get history and ensure that the expected file has been ingested
-    getHistoryAndAssertLoad(manager, TEST_FILE_NAME);
-
-    IngestResponse insertResponseSkippedFiles = manager.ingestFile(myFile, null, true);
-
-    assertEquals("SUCCESS", insertResponseSkippedFiles.getResponseCode());
-    assertEquals(1, insertResponseSkippedFiles.getSkippedFiles().size());
-    assertEquals(
-        TEST_FILE_NAME, insertResponseSkippedFiles.getSkippedFiles().stream().findFirst().get());
+    testAndVerifySimpleIngestionForOnePipe(manager);
   }
 
   /** ingest test example ingest a simple file and check load history. */
@@ -301,6 +281,69 @@ public void testUserAgentSuffixForInsertFileAPI() throws Exception {
     verifyDefaultUserAgent(noUserAgentUsed.getAllHeaders(), false, null);
   }
 
+  /**
+   * Creates multiple pipes with same stage and table to verify behavior of HttpUtil and BG thread
+   * for ConnectionPoolingManager
+   *
+   * <p>Creates the thread only two times since the manager was closed only once.
+   */
+  @Test
+  public void testMultipleSimpleIngestManagers() throws Exception {
+    // put
+    TestUtils.executeQuery("put file://" + testFilePath + " @" + stageName);
+
+    // create ingest manager
+    SimpleIngestManager manager = TestUtils.getManager(pipeName);
+
+    manager.close();
+
+    final String pipeName2 = PIPE_NAME_PREFIX + "" + (RAND_NUM + 1);
+    createPipe(tableName, stageName, pipeName2);
+
+    SimpleIngestManager manager2 = TestUtils.getManager(pipeName2);
+
+    testAndVerifySimpleIngestionForOnePipe(manager2);
+
+    final String pipeName3 = PIPE_NAME_PREFIX + "" + (RAND_NUM + 2);
+    createPipe(tableName, stageName, pipeName3);
+
+    // creating one more SimpleIngestManager
+    SimpleIngestManager manager3 = TestUtils.getManager(pipeName3);
+    testAndVerifySimpleIngestionForOnePipe(manager3);
+  }
+
+  private static void createPipe(
+      final String tableName, final String stageName, final String pipeName) throws Exception {
+    TestUtils.executeQuery(
+        "create or replace pipe "
+            + pipeName
+            + " as copy into "
+            + tableName
+            + " from @"
+            + stageName);
+  }
+
+  private void testAndVerifySimpleIngestionForOnePipe(final SimpleIngestManager simpleIngestManager)
+      throws Exception {
+    // create a file wrapper
+    StagedFileWrapper myFile = new StagedFileWrapper(TEST_FILE_NAME, null);
+
+    // get an insert response after we submit
+    IngestResponse insertResponse = simpleIngestManager.ingestFile(myFile, null);
+
+    assertEquals("SUCCESS", insertResponse.getResponseCode());
+
+    // Get history and ensure that the expected file has been ingested
+    getHistoryAndAssertLoad(simpleIngestManager, TEST_FILE_NAME);
+
+    IngestResponse insertResponseSkippedFiles = simpleIngestManager.ingestFile(myFile, null, true);
+
+    assertEquals("SUCCESS", insertResponseSkippedFiles.getResponseCode());
+    assertEquals(1, insertResponseSkippedFiles.getSkippedFiles().size());
+    assertEquals(
+        TEST_FILE_NAME, insertResponseSkippedFiles.getSkippedFiles().stream().findFirst().get());
+  }
+
   private void verifyDefaultUserAgent(
       final Header[] headers,
       final boolean verifyAdditionalUserAgentInfo,

From b20d5126dbbdeca940c431d4ba9edfb066952d7c Mon Sep 17 00:00:00 2001
From: Jay Patel <jay.patel@snowflake.com>
Date: Wed, 13 Jul 2022 17:40:46 -0700
Subject: [PATCH 043/356] NO-SNOW check the retry logic with millis instead of
 ms and seconds (#197)

* NO-SNOW check the retry logic with millis instead of ms and seconds

* Add comment on timeoutexception test

* Added a test to erify the retry behavior, but ignored
---
 .../streaming/internal/RegisterService.java   |  3 +-
 .../internal/RegisterServiceTest.java         | 49 +++++++++++++++++++
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index bbcd2fecf..f2eca8c8f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -111,7 +111,8 @@ List<FlushService.BlobData> registerBlobs(Map<String, Timer.Context> latencyTime
           List<BlobMetadata> blobs = new ArrayList<>();
           long startTime = System.currentTimeMillis();
           while (idx < oldList.size()
-              && System.currentTimeMillis() - startTime <= BLOB_UPLOAD_TIMEOUT_IN_SEC * 2) {
+              && System.currentTimeMillis() - startTime
+                  <= TimeUnit.SECONDS.toMillis(BLOB_UPLOAD_TIMEOUT_IN_SEC * 2)) {
             Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>> futureBlob =
                 oldList.get(idx);
             try {
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
index 4de405a8f..d8cda59be 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
@@ -1,13 +1,17 @@
 package net.snowflake.ingest.streaming.internal;
 
+import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import net.snowflake.ingest.utils.Pair;
 import org.junit.Assert;
+import org.junit.Ignore;
 import org.junit.Test;
 
 public class RegisterServiceTest {
@@ -27,6 +31,15 @@ public void testRegisterService() throws Exception {
     Assert.assertEquals(0, errorBlobs.size());
   }
 
+  /**
+   * Note that this exception will not perform retries since the completeExceptionally throws the
+   * exception by wrapping inside ExecutionException. Check method {@link
+   * CompletableFuture#get(long, TimeUnit)} javadocs
+   *
+   * <p>The check for retries checks the original exception instead of the exception.getCause()
+   *
+   * @throws Exception
+   */
   @Test
   public void testRegisterServiceTimeoutException() throws Exception {
     SnowflakeStreamingIngestClientInternal client =
@@ -53,6 +66,42 @@ public void testRegisterServiceTimeoutException() throws Exception {
     }
   }
 
+  // Ignore since it runs for BLOB_UPLOAD_TIMEOUT_IN_SEC * BLOB_UPLOAD_MAX_RETRY_COUNT
+  @Ignore
+  @Test
+  public void testRegisterServiceTimeoutException_testRetries() throws Exception {
+    SnowflakeStreamingIngestClientInternal client =
+        new SnowflakeStreamingIngestClientInternal("client");
+    RegisterService rs = new RegisterService(client, true);
+
+    Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>> blobFuture1 =
+        new Pair<>(
+            new FlushService.BlobData("success", new ArrayList<>()),
+            CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null)));
+    CompletableFuture future = new CompletableFuture();
+    future.thenRunAsync(
+        () -> {
+          try {
+            Thread.sleep(TimeUnit.SECONDS.toMillis(BLOB_UPLOAD_TIMEOUT_IN_SEC) + 5);
+          } catch (InterruptedException e) {
+            e.printStackTrace();
+          }
+          return;
+        });
+    Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>> blobFuture2 =
+        new Pair<>(new FlushService.BlobData("fail", new ArrayList<>()), future);
+    rs.addBlobs(Arrays.asList(blobFuture1, blobFuture2));
+    Assert.assertEquals(2, rs.getBlobsList().size());
+    try {
+      List<FlushService.BlobData> errorBlobs = rs.registerBlobs(null);
+      Assert.assertEquals(0, rs.getBlobsList().size());
+      Assert.assertEquals(1, errorBlobs.size());
+      Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
+    } catch (Exception e) {
+      Assert.fail("The timeout exception should be caught in registerBlobs");
+    }
+  }
+
   @Test
   public void testRegisterServiceNonTimeoutException() {
     SnowflakeStreamingIngestClientInternal client =

From 4cdcae940ffcd045a578a7bcb9a3f2d6a9fb8f59 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 14 Jul 2022 17:06:15 -0700
Subject: [PATCH 044/356] SNOW-626104: Support ARRAY data type in the SDK
 (#196)

Currently, we're not handling the ARRAY data type correctly in the SDK, this PR fixes the issue by adding the ARRAY specific parsing logic.
---
 .../streaming/internal/ArrowRowBuffer.java    |  7 +++
 .../internal/DataValidationUtil.java          | 50 ++++++++++++++++++-
 .../internal/DataValidationUtilTest.java      | 35 +++++++++++++
 3 files changed, 91 insertions(+), 1 deletion(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 46faa1fb2..bfe27cf94 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -767,6 +767,13 @@ private float convertRowToArrow(
               break;
             }
           case ARRAY:
+            {
+              String str = DataValidationUtil.validateAndParseArray(value);
+              Text text = new Text(str);
+              ((VarCharVector) vector).setSafe(curRowIndex, text);
+              rowBufferSize += text.getBytes().length;
+              break;
+            }
           case VARIANT:
             {
               String str = DataValidationUtil.validateAndParseVariant(value);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 190a69250..2fec5b4e1 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -8,6 +8,8 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -42,7 +44,7 @@ static String validateAndParseVariant(Object input) {
       output = input.toString();
     } catch (Exception e) {
       throw new SFException(
-          e, ErrorCode.INVALID_ROW, input.toString(), "Input column can't be convert to String.");
+          e, ErrorCode.INVALID_ROW, input, "Input column can't be convert to String.");
     }
 
     if (output.length() > MAX_STRING_LENGTH) {
@@ -55,6 +57,52 @@ static String validateAndParseVariant(Object input) {
     return output;
   }
 
+  /**
+   * Expects an Array or List object
+   *
+   * @param input the input data, must be able to convert to String
+   */
+  static String validateAndParseArray(Object input) {
+    if (!input.getClass().isArray() && !(input instanceof List)) {
+      throw new SFException(
+          ErrorCode.INVALID_ROW, input, "Input column must be an Array or List object.");
+    }
+
+    String output;
+
+    try {
+      if (input.getClass().isArray()) {
+        if (input instanceof int[]) {
+          output = Arrays.toString((int[]) input);
+        } else if (input instanceof long[]) {
+          output = Arrays.toString((long[]) input);
+        } else if (input instanceof short[]) {
+          output = Arrays.toString((short[]) input);
+        } else if (input instanceof double[]) {
+          output = Arrays.toString((double[]) input);
+        } else if (input instanceof float[]) {
+          output = Arrays.toString((float[]) input);
+        } else if (input instanceof byte[]) {
+          output = Arrays.toString((byte[]) input);
+        } else if (input instanceof char[]) {
+          output = Arrays.toString((char[]) input);
+        } else if (input instanceof boolean[]) {
+          output = Arrays.toString((boolean[]) input);
+        } else if (input instanceof Object[]) {
+          output = Arrays.deepToString((Object[]) input);
+        } else {
+          throw new SFException(ErrorCode.INVALID_ROW, input, "Input array type is not supported.");
+        }
+      } else {
+        output = input.toString();
+      }
+    } catch (Exception e) {
+      throw new SFException(
+          e, ErrorCode.INVALID_ROW, input, "Input column can't be convert to String.");
+    }
+    return output;
+  }
+
   /**
    * Expects string JSON or JsonNode
    *
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 01ff6c7c6..0d79d1bf6 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -9,6 +9,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Function;
@@ -263,6 +264,40 @@ public void testValidateAndParseVariant() throws Exception {
     }
   }
 
+  @Test
+  public void testValidateAndParseArray() throws Exception {
+    int invalidArray = 1;
+    try {
+      DataValidationUtil.validateAndParseArray(invalidArray);
+      Assert.fail("Expected INVALID_ROW error");
+    } catch (SFException err) {
+      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), err.getVendorCode());
+    }
+
+    int[] intArray = new int[] {1, 2, 3};
+    Assert.assertEquals("[1, 2, 3]", DataValidationUtil.validateAndParseArray(intArray));
+
+    Object[] objectArray = new Object[] {1, 2, 3};
+    Assert.assertEquals("[1, 2, 3]", DataValidationUtil.validateAndParseArray(objectArray));
+
+    Object[] ObjectArrayWithNull = new Object[] {1, null, 3};
+    Assert.assertEquals(
+        "[1, null, 3]", DataValidationUtil.validateAndParseArray(ObjectArrayWithNull));
+
+    Object[][] nestedArray = new Object[][] {{1, 2, 3}, null, {4, 5, 6}};
+    Assert.assertEquals(
+        "[[1, 2, 3], null, [4, 5, 6]]", DataValidationUtil.validateAndParseArray(nestedArray));
+
+    List<Integer> intList = Arrays.asList(1, 2, 3);
+    Assert.assertEquals("[1, 2, 3]", DataValidationUtil.validateAndParseArray(intList));
+
+    List<Object> objectList = Arrays.asList(1, 2, 3);
+    Assert.assertEquals("[1, 2, 3]", DataValidationUtil.validateAndParseArray(objectList));
+
+    List<Object> nestedList = Arrays.asList(Arrays.asList(1, 2, 3), 2, 3);
+    Assert.assertEquals("[[1, 2, 3], 2, 3]", DataValidationUtil.validateAndParseArray(nestedList));
+  }
+
   @Test
   public void testValidateAndParseObject() throws Exception {
     String stringObject = "{\"key\":1}";

From 90eb5e5fc7f3c52b94b67f25649d871101b1dace Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Fri, 15 Jul 2022 11:48:30 -0700
Subject: [PATCH 045/356] NO-SNOW: fix logging format to NOT add a new line 
 (#198)

For more detail, please look at snowflakedb/snowflake-kafka-connector#87
---
 src/main/java/net/snowflake/ingest/utils/Logging.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/net/snowflake/ingest/utils/Logging.java b/src/main/java/net/snowflake/ingest/utils/Logging.java
index b44f2c852..2d5f90be0 100644
--- a/src/main/java/net/snowflake/ingest/utils/Logging.java
+++ b/src/main/java/net/snowflake/ingest/utils/Logging.java
@@ -108,7 +108,7 @@ public Logger getLogger() {
    * @return log message wrapped by snowflake tag
    */
   private static String logMessage(String msg) {
-    return "\n".concat(msg).replaceAll("\n", "\n" + SF_LOG_TAG + " ");
+    return SF_LOG_TAG + " " + msg;
   }
 
   /**

From d299a4ed7af799f855edab0c4e84e474a1e3c6b5 Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Fri, 8 Apr 2022 11:45:30 +0000
Subject: [PATCH 046/356] @snow SNOW-612516 Streaming ingest: add random file
 reader to scan single columns, format version 2

---
 pom.xml                                       |  10 +-
 scripts/check_content.sh                      |   2 +-
 .../ArrowFileWriterWithCompression.java       |  65 ++++++
 .../streaming/internal/BlobBuilder.java       |  44 +++-
 .../streaming/internal/BlobMetadata.java      |  20 ++
 .../internal/CustomCompressionCodec.java      | 108 ++++++++++
 .../streaming/internal/FlushService.java      |  67 ++++--
 ...nowflakeStreamingIngestClientInternal.java |   6 +-
 .../net/snowflake/ingest/utils/Constants.java |  59 ++++-
 .../ingest/utils/ParameterProvider.java       |  24 +++
 .../java/net/snowflake/ingest/TestUtils.java  |   9 +
 .../streaming/internal/FlushServiceTest.java  |   9 +-
 .../SnowflakeStreamingIngestClientTest.java   |   1 +
 .../streaming/internal/StreamingIngestIT.java | 204 ++++++++++++++++--
 14 files changed, 580 insertions(+), 48 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ArrowFileWriterWithCompression.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/CustomCompressionCodec.java

diff --git a/pom.xml b/pom.xml
index 248576323..d2599b287 100644
--- a/pom.xml
+++ b/pom.xml
@@ -42,6 +42,7 @@
         <shadeBase>net.snowflake.ingest.internal</shadeBase>
         <jacoco.skip.instrument>true</jacoco.skip.instrument>
         <jacoco.version>0.8.5</jacoco.version>
+        <arrow.version>8.0.0</arrow.version>
     </properties>
 
     <build>
@@ -319,14 +320,19 @@
         <dependency>
             <groupId>org.apache.arrow</groupId>
             <artifactId>arrow-vector</artifactId>
-            <version>8.0.0</version>
+            <version>${arrow.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.arrow</groupId>
             <artifactId>arrow-memory-netty</artifactId>
-            <version>8.0.0</version>
+            <version>${arrow.version}</version>
             <scope>runtime</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-compression</artifactId>
+            <version>${arrow.version}</version>
+        </dependency>
 
         <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-core -->
         <dependency>
diff --git a/scripts/check_content.sh b/scripts/check_content.sh
index b25f24901..9e5b1c404 100755
--- a/scripts/check_content.sh
+++ b/scripts/check_content.sh
@@ -5,7 +5,7 @@
 set -o pipefail
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-if jar tvf $DIR/../target/snowflake-ingest-sdk.jar  | awk '{print $8}' | grep -v -E "^(net|com)/snowflake" | grep -v -E "(com|net)/\$" | grep -v -E "^META-INF" | grep -v -E "^mozilla" | grep -v mime.types | grep -v project.properties | grep -v -E "javax" | grep -v -E "^org/" | grep -v -E "^com/google" | grep -v -E "^com/sun" | grep -v "log4j.properties" | grep -v "git.properties" | grep -v "io/" | grep -v "codegen/" | grep -v "com/codahale/" | grep -v "com/ibm/" | grep -v "LICENSE"; then
+if jar tvf $DIR/../target/snowflake-ingest-sdk.jar  | awk '{print $8}' | grep -v -E "^(net|com)/snowflake" | grep -v -E "(com|net)/\$" | grep -v -E "^META-INF" | grep -v -E "^mozilla" | grep -v mime.types | grep -v project.properties | grep -v -E "javax" | grep -v -E "^org/" | grep -v -E "^com/google" | grep -v -E "^com/sun" | grep -v "log4j.properties" | grep -v "git.properties" | grep -v "io/" | grep -v "codegen/" | grep -v "com/codahale/" | grep -v "com/ibm/" | grep -v "LICENSE" | grep -v "aix/" | grep -v "darwin/" | grep -v "win/" | grep -v "freebsd/" | grep -v "linux/" | grep -v "com/github/"; then
   echo "[ERROR] Ingest SDK jar includes class not under the snowflake namespace"
   exit 1
 fi
\ No newline at end of file
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFileWriterWithCompression.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFileWriterWithCompression.java
new file mode 100644
index 000000000..a07509e7c
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFileWriterWithCompression.java
@@ -0,0 +1,65 @@
+package net.snowflake.ingest.streaming.internal;
+
+import java.io.IOException;
+import java.nio.channels.WritableByteChannel;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.compression.CompressionCodec;
+import org.apache.arrow.vector.compression.NoCompressionCodec;
+import org.apache.arrow.vector.ipc.ArrowFileWriter;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
+
+/**
+ * {@link ArrowFileWriter} with configurable {@link CompressionCodec}
+ *
+ * <p>{@link ArrowFileWriter} cannot be configured to use a custom {@link CompressionCodec}. {@link
+ * ArrowFileWriter} internally has a {@link VectorUnloader} that can be configured with a custom
+ * {@link CompressionCodec} but {@link ArrowFileWriter} does not forward this configuration in its
+ * public constructors. This class extends {@link ArrowFileWriter} to override the underlying {@link
+ * VectorUnloader} and configure it with a custom {@link CompressionCodec}.
+ */
+class ArrowFileWriterWithCompression extends ArrowFileWriter {
+  private final VectorUnloader unloader;
+  private final boolean compressed;
+
+  ArrowFileWriterWithCompression(
+      VectorSchemaRoot root, WritableByteChannel out, CompressionCodec codec) {
+    // Note: the dictionary provider is null, this allows to simplify writeBatch() in this class
+    // and exclude ensureDictionariesWritten() call from the original ArrowWriter#writeBatch()
+    // implementation
+    super(root, null, out);
+
+    // we have to create an unloader, different to ArrowWriter#unloader
+    // because it is private and we need to configure it with CompressionCodec
+    // that is not available in ArrowWriter
+    this.unloader = new VectorUnloader(root, true, codec, true);
+    this.compressed = codec != NoCompressionCodec.INSTANCE;
+  }
+
+  // same as ArrowWriter#writeBatch but does not write dictionaries
+  // and uses an unloader configured with a custom CompressionCodec
+  @Override
+  public void writeBatch() throws IOException {
+    start(); // same as ensureStarted() in ArrowWriter#writeBatch()
+
+    // not called as in ArrowWriter#writeBatch
+    // because the dictionary provider is always null in ArrowFileWriterWithCompression()
+    // constructor
+    // ensureDictionariesWritten();
+
+    // ArrowWriter#unloader is private but we need to configure it with CompressionCodec
+    // hence we use here a different unloader
+    // the compression happenes in unloader.getRecordBatch() -> appendNodes() ->
+    // buffers.add(codec.compress(..)..)
+    try (ArrowRecordBatch batch = unloader.getRecordBatch()) {
+      writeRecordBatch(batch);
+      if (compressed) {
+        // compression creates new compressed buffers added to the ArrowRecordBatch
+        // their reference counter is incremented twice:
+        // once in CompressionCodec on create and once in ArrowRecordBatch to retain
+        // first the buffers are released here, second in the ArrowRecordBatch#close on try exit
+        batch.getBuffers().forEach(arrowBuf -> arrowBuf.getReferenceManager().release());
+      }
+    }
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
index da1f3dfea..96a1ef635 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
@@ -8,7 +8,6 @@
 import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
 import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
 import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_FORMAT_VERSION;
 import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
 import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
 import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
@@ -24,6 +23,7 @@
 import java.util.List;
 import java.util.zip.GZIPOutputStream;
 import javax.xml.bind.DatatypeConverter;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.Pair;
 
@@ -95,6 +95,42 @@ static Pair<byte[], Integer> compress(
     return new Pair<>(compressedOutputStream.toByteArray(), compressedSize);
   }
 
+  /**
+   * Gzip compress the given chunk data if required by the given write mode and pads the compressed
+   * data for encryption.
+   *
+   * @param filePath blob file full path
+   * @param chunkData uncompressed chunk data
+   * @param blockSizeToAlignTo block size to align to for encryption
+   * @param arrowBatchWriteMode Arrow format write mode
+   * @return padded compressed chunk data, aligned to blockSizeToAlignTo, and actual length of
+   *     compressed data before padding at the end
+   * @throws IOException
+   */
+  static Pair<byte[], Integer> compressIfNeededAndPadChunk(
+      String filePath,
+      ByteArrayOutputStream chunkData,
+      int blockSizeToAlignTo,
+      Constants.ArrowBatchWriteMode arrowBatchWriteMode)
+      throws IOException {
+    // Encryption needs padding to the ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES
+    // to align with decryption on the Snowflake query path starting from this chunk offset.
+    // The padding does not have arrow data and not compressed.
+    // Hence, the actual chunk size is smaller by the padding size.
+    // The compression on the Snowflake query path needs the correct size of the compressed
+    // data.
+    if (arrowBatchWriteMode == Constants.ArrowBatchWriteMode.STREAM) {
+      // Stream write mode does not support column level compression.
+      // Compress the chunk data and pad it for encryption.
+      return BlobBuilder.compress(filePath, chunkData, blockSizeToAlignTo);
+    } else {
+      int actualSize = chunkData.size();
+      int paddingSize = blockSizeToAlignTo - actualSize % blockSizeToAlignTo;
+      chunkData.write(new byte[paddingSize]);
+      return new Pair<>(chunkData.toByteArray(), actualSize);
+    }
+  }
+
   /**
    * Build the blob file
    *
@@ -102,6 +138,7 @@ static Pair<byte[], Integer> compress(
    * @param chunksDataList List of chunk data
    * @param chunksChecksum Checksum for the chunk data portion
    * @param chunksDataSize Total size for the chunk data portion after compression
+   * @param bdecVersion BDEC file version
    * @return The blob file as a byte array
    * @throws JsonProcessingException
    */
@@ -109,7 +146,8 @@ static byte[] build(
       List<ChunkMetadata> chunksMetadataList,
       List<byte[]> chunksDataList,
       long chunksChecksum,
-      long chunksDataSize)
+      long chunksDataSize,
+      Constants.BdecVerion bdecVersion)
       throws IOException {
     byte[] chunkMetadataListInBytes = MAPPER.writeValueAsBytes(chunksMetadataList);
 
@@ -127,7 +165,7 @@ static byte[] build(
     ByteArrayOutputStream blob = new ByteArrayOutputStream();
     if (!BLOB_NO_HEADER) {
       blob.write(BLOB_EXTENSION_TYPE.getBytes());
-      blob.write(BLOB_FORMAT_VERSION);
+      blob.write(bdecVersion.toByte());
       blob.write(toByteArray(metadataSize + chunksDataSize));
       blob.write(toByteArray(chunksChecksum));
       blob.write(toByteArray(chunkMetadataListInBytes.length));
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index e1dbdf764..7ae2eca76 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -4,21 +4,36 @@
 
 package net.snowflake.ingest.streaming.internal;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import java.util.List;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ParameterProvider;
 
 /** Metadata for a blob that sends to Snowflake as part of the register blob request */
 class BlobMetadata {
   private final String path;
   private final String md5;
+  private final Constants.BdecVerion bdecVersion;
   private final List<ChunkMetadata> chunks;
 
   BlobMetadata(String path, String md5, List<ChunkMetadata> chunks) {
+    this(path, md5, ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT, chunks);
+  }
+
+  BlobMetadata(
+      String path, String md5, Constants.BdecVerion bdecVersion, List<ChunkMetadata> chunks) {
     this.path = path;
     this.md5 = md5;
+    this.bdecVersion = bdecVersion;
     this.chunks = chunks;
   }
 
+  @JsonIgnore
+  Constants.BdecVerion getVersion() {
+    return bdecVersion;
+  }
+
   @JsonProperty("path")
   String getPath() {
     return this.path;
@@ -33,4 +48,9 @@ String getMD5() {
   List<ChunkMetadata> getChunks() {
     return this.chunks;
   }
+
+  @JsonProperty("bdec_version")
+  byte getVersionByte() {
+    return bdecVersion.toByte();
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/CustomCompressionCodec.java b/src/main/java/net/snowflake/ingest/streaming/internal/CustomCompressionCodec.java
new file mode 100644
index 000000000..8c668b114
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/CustomCompressionCodec.java
@@ -0,0 +1,108 @@
+package net.snowflake.ingest.streaming.internal;
+
+import org.apache.arrow.compression.ZstdCompressionCodec;
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.util.MemoryUtil;
+import org.apache.arrow.vector.compression.CompressionCodec;
+import org.apache.arrow.vector.compression.CompressionUtil;
+
+/**
+ * This is custom wrapper on top of {@link CompressionCodec} implementations.
+ *
+ * <p>The main stream {@link CompressionCodec} implementations are based on {@link
+ * org.apache.arrow.vector.compression.AbstractCompressionCodec}. This implementation is similar to
+ * it with some tweaks.
+ *
+ * <p>The reason to have this custom implementation is to fix certain issues with mainstream codecs
+ * that lead to buffer reference counting problems due to creation of new compressed buffers and
+ * releasing uncompressed. The release of uncompressed buffer from the original vectors to write
+ * into file results in double release when the original vector gets released as usual.
+ *
+ * <p>The custom code extraction is not optimal in a long term and there is a TODO: SNOW-629844 - to
+ * follow up on this.
+ */
+public class CustomCompressionCodec implements CompressionCodec {
+  private final CompressionCodec actualCodec;
+
+  public CustomCompressionCodec(CompressionUtil.CodecType codecType) {
+    switch (codecType) {
+        // TODO: SNOW-629836 - LZ4 compression currently breaks server side decompression
+      case ZSTD:
+        actualCodec =
+            new ZstdCompressionCodec() {
+              @Override
+              public ArrowBuf compress(BufferAllocator allocator, ArrowBuf uncompressedBuffer) {
+                return doCompress(allocator, uncompressedBuffer);
+              }
+            };
+        break;
+      case NO_COMPRESSION:
+        actualCodec = new CustomNoCompressionCodec();
+        break;
+      default:
+        throw new IllegalArgumentException("unsupported compression type: " + codecType);
+    }
+  }
+
+  @Override
+  public ArrowBuf compress(BufferAllocator allocator, ArrowBuf uncompressedBuffer) {
+    if (uncompressedBuffer.writerIndex() == 0L) {
+      // shortcut for empty buffer
+      ArrowBuf compressedBuffer = allocator.buffer(CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH);
+      compressedBuffer.setLong(0, 0);
+      compressedBuffer.writerIndex(CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH);
+      uncompressedBuffer.close();
+      return compressedBuffer;
+    }
+
+    ArrowBuf compressedBuffer = actualCodec.compress(allocator, uncompressedBuffer);
+    // Here we simplify the original AbstractCompressionCodec#compress
+    // and always write the compressed data even if the compression does not give anything
+    // in case of e.g. random integers.
+    // The reason is that mixing compressed and uncompressed buffers breaks the server side
+    // decompression that happens to expect only compressed buffers for version 8.0.0.
+    long uncompressedLength = uncompressedBuffer.writerIndex();
+    writeUncompressedLength(compressedBuffer, uncompressedLength);
+
+    // Here we again simplify the original AbstractCompressionCodec#compress
+    // and do not release the uncompressedBuffer because it is part of the root vector to write.
+    // This root vector will be released as usual by client.
+    return compressedBuffer;
+  }
+
+  @Override
+  public ArrowBuf decompress(BufferAllocator allocator, ArrowBuf compressedBuffer) {
+    throw new UnsupportedOperationException("decompress is not supported");
+  }
+
+  protected void writeUncompressedLength(ArrowBuf compressedBuffer, long uncompressedLength) {
+    if (!MemoryUtil.LITTLE_ENDIAN) {
+      uncompressedLength = Long.reverseBytes(uncompressedLength);
+    }
+    // first 8 bytes reserved for uncompressed length, according to the specification
+    compressedBuffer.setLong(0, uncompressedLength);
+  }
+
+  @Override
+  public CompressionUtil.CodecType getCodecType() {
+    return actualCodec.getCodecType();
+  }
+
+  private static class CustomNoCompressionCodec implements CompressionCodec {
+    @Override
+    public ArrowBuf compress(BufferAllocator allocator, ArrowBuf uncompressedBuffer) {
+      return CompressionUtil.packageRawBuffer(allocator, uncompressedBuffer);
+    }
+
+    @Override
+    public ArrowBuf decompress(BufferAllocator allocator, ArrowBuf compressedBuffer) {
+      return compressedBuffer;
+    }
+
+    @Override
+    public CompressionUtil.CodecType getCodecType() {
+      return CompressionUtil.CodecType.NO_COMPRESSION;
+    }
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 85c9c61e4..4adbcd764 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -15,6 +15,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
+import java.nio.channels.Channels;
 import java.security.InvalidAlgorithmParameterException;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
@@ -50,7 +51,9 @@
 import org.apache.arrow.vector.VectorLoader;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.compression.CompressionUtil;
 import org.apache.arrow.vector.ipc.ArrowStreamWriter;
+import org.apache.arrow.vector.ipc.ArrowWriter;
 import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
 
 /**
@@ -121,6 +124,9 @@ List<List<ChannelData>> getData() {
   // A map which stores the timer for each blob in order to capture the flush latency
   private final Map<String, Timer.Context> latencyTimerContextMap;
 
+  // blob file version
+  private final Constants.BdecVerion bdecVersion;
+
   /**
    * Constructor for TESTING that takes (usually mocked) StreamingIngestStage
    *
@@ -142,6 +148,7 @@ List<List<ChannelData>> getData() {
     this.lastFlushTime = System.currentTimeMillis();
     this.isTestMode = isTestMode;
     this.latencyTimerContextMap = new HashMap<>();
+    this.bdecVersion = this.owningClient.getParameterProvider().getBlobFormatVersion();
     createWorkers();
   }
 
@@ -174,6 +181,7 @@ List<List<ChannelData>> getData() {
     this.lastFlushTime = System.currentTimeMillis();
     this.isTestMode = isTestMode;
     this.latencyTimerContextMap = new HashMap<>();
+    this.bdecVersion = this.owningClient.getParameterProvider().getBlobFormatVersion();
     createWorkers();
   }
 
@@ -403,7 +411,7 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
       List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
       long rowCount = 0L;
       VectorSchemaRoot root = null;
-      ArrowStreamWriter streamWriter = null;
+      ArrowWriter arrowWriter = null;
       VectorLoader loader = null;
       SnowflakeStreamingIngestChannelInternal firstChannel = null;
       ByteArrayOutputStream chunkData = new ByteArrayOutputStream();
@@ -431,10 +439,16 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
           if (root == null) {
             columnEpStatsMapCombined = data.getColumnEps();
             root = data.getVectors();
-            streamWriter = new ArrowStreamWriter(root, null, chunkData);
+            arrowWriter =
+                getArrowBatchWriteMode() == Constants.ArrowBatchWriteMode.STREAM
+                    ? new ArrowStreamWriter(root, null, chunkData)
+                    : new ArrowFileWriterWithCompression(
+                        root,
+                        Channels.newChannel(chunkData),
+                        new CustomCompressionCodec(CompressionUtil.CodecType.ZSTD));
             loader = new VectorLoader(root);
             firstChannel = data.getChannel();
-            streamWriter.start();
+            arrowWriter.start();
           } else {
             // This method assumes that channelsDataPerTable is grouped by table. We double check
             // here and throw an error if the assumption is violated
@@ -456,7 +470,7 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
           }
 
           // Write channel data using the stream writer
-          streamWriter.writeBatch();
+          arrowWriter.writeBatch();
           rowCount += data.getRowCount();
 
           logger.logDebug(
@@ -467,23 +481,19 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
               filePath);
         }
       } finally {
-        if (streamWriter != null) {
-          streamWriter.close();
+        if (arrowWriter != null) {
+          arrowWriter.close();
           root.close();
         }
       }
 
       if (!channelsMetadataList.isEmpty()) {
-        // Compress the chunk data and pad it for encryption.
-        // Encryption needs padding to the ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES
-        // to align with decryption on the Snowflake query path starting from this chunk offset.
-        // The padding does not have arrow data and not compressed.
-        // Hence, the actual chunk size is smaller by the padding size.
-        // The compression on the Snowflake query path needs the correct size of the compressed
-        // data.
         Pair<byte[], Integer> compressionResult =
-            BlobBuilder.compress(
-                filePath, chunkData, Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES);
+            BlobBuilder.compressIfNeededAndPadChunk(
+                filePath,
+                chunkData,
+                Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES,
+                getArrowBatchWriteMode());
         byte[] compressedAndPaddedChunkData = compressionResult.getFirst();
         int compressedChunkLength = compressionResult.getSecond();
 
@@ -502,12 +512,13 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
         int encryptedCompressedChunkDataSize = encryptedCompressedChunkData.length;
 
         // Create chunk metadata
+        long startOffset = curDataSize;
         ChunkMetadata chunkMetadata =
             ChunkMetadata.builder()
                 .setOwningTable(firstChannel)
                 // The start offset will be updated later in BlobBuilder#build to include the blob
                 // header
-                .setChunkStartOffset(curDataSize)
+                .setChunkStartOffset(startOffset)
                 // The compressedChunkLength is used because it is the actual data size used for
                 // decompression and md5 calculation on server side.
                 .setChunkLength(compressedChunkLength)
@@ -524,21 +535,24 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
         crc.update(encryptedCompressedChunkData, 0, encryptedCompressedChunkDataSize);
 
         logger.logInfo(
-            "Finish building chunk in blob={}, table={}, rowCount={}, uncompressedSize={},"
-                + " compressedChunkLength={}, encryptedCompressedSize={}",
+            "Finish building chunk in blob={}, table={}, rowCount={}, startOffset={},"
+                + " uncompressedSize={}, compressedChunkLength={}, encryptedCompressedSize={},"
+                + " arrowBatchWriteMode={}",
             filePath,
             firstChannel.getFullyQualifiedTableName(),
             rowCount,
+            startOffset,
             chunkData.size(),
             compressedChunkLength,
             encryptedCompressedChunkDataSize,
-            compressedChunkLength);
+            getArrowBatchWriteMode());
       }
     }
 
     // Build blob file, and then upload to streaming ingest dedicated stage
     byte[] blob =
-        BlobBuilder.build(chunksMetadataList, chunksDataList, crc.getValue(), curDataSize);
+        BlobBuilder.build(
+            chunksMetadataList, chunksDataList, crc.getValue(), curDataSize, bdecVersion);
     if (buildContext != null) {
       buildContext.stop();
     }
@@ -546,6 +560,17 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
     return upload(filePath, blob, chunksMetadataList);
   }
 
+  private Constants.ArrowBatchWriteMode getArrowBatchWriteMode() {
+    switch (bdecVersion) {
+      case ONE:
+        return Constants.ArrowBatchWriteMode.STREAM;
+      case TWO:
+        return Constants.ArrowBatchWriteMode.FILE;
+      default:
+        throw new IllegalArgumentException("Unsupported BLOB_FORMAT_VERSION: " + bdecVersion);
+    }
+  }
+
   /**
    * Upload a blob to Streaming Ingest dedicated stage
    *
@@ -577,7 +602,7 @@ BlobMetadata upload(String filePath, byte[] blob, List<ChunkMetadata> metadata)
         blob.length,
         System.currentTimeMillis() - startTime);
 
-    return new BlobMetadata(filePath, BlobBuilder.computeMD5(blob), metadata);
+    return new BlobMetadata(filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 2787c9088..aec8311ce 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -531,7 +531,11 @@ List<BlobMetadata> getRetryBlobs(
                   .collect(Collectors.toList());
           if (!relevantChunks.isEmpty()) {
             retryBlobs.add(
-                new BlobMetadata(blobMetadata.getPath(), blobMetadata.getMD5(), relevantChunks));
+                new BlobMetadata(
+                    blobMetadata.getPath(),
+                    blobMetadata.getMD5(),
+                    blobMetadata.getVersion(),
+                    relevantChunks));
           }
         });
 
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index bd935ec36..bd1691efe 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -4,6 +4,8 @@
 
 package net.snowflake.ingest.utils;
 
+import java.util.Arrays;
+
 /** Contains all the constants needed for Streaming Ingest */
 public class Constants {
 
@@ -33,7 +35,6 @@ public class Constants {
   public static final int INSERT_THROTTLE_MAX_RETRY_COUNT = 10;
   public static final long MAX_BLOB_SIZE_IN_BYTES = 512000000L;
   public static final long MAX_CHUNK_SIZE_IN_BYTES = 32000000L;
-  public static final byte BLOB_FORMAT_VERSION = 0;
   public static final int BLOB_TAG_SIZE_IN_BYTES = 4;
   public static final int BLOB_VERSION_SIZE_IN_BYTES = 1;
   public static final int BLOB_FILE_SIZE_SIZE_IN_BYTES = 8;
@@ -55,11 +56,65 @@ public class Constants {
   public static final String OPEN_CHANNEL_ENDPOINT = "/v1/streaming/channels/open/";
   public static final String REGISTER_BLOB_ENDPOINT = "/v1/streaming/channels/write/blobs/";
 
-  public static enum WriteMode {
+  public enum WriteMode {
     CLOUD_STORAGE,
     REST_API,
   }
 
+  /** Thw write mode to generate Arrow BDEC file. */
+  public enum ArrowBatchWriteMode {
+    /** Stream format is produced by {@link org.apache.arrow.vector.ipc.ArrowStreamWriter}. */
+    STREAM,
+
+    /**
+     * File format is produced by {@link org.apache.arrow.vector.ipc.ArrowFileWriter}.
+     *
+     * <p>The file format is same as stream format but it adds a footer at the end of the file. The
+     * footer contains metadata for quick random access of certain column data in batches when it is
+     * being read on server side. This way there is no need to download and parse the whole file if
+     * only certain columns are requested.
+     */
+    FILE,
+  }
+
+  /** Thw write mode to generate Arrow BDEC file. */
+  public enum BdecVerion {
+    /** Uses Arrow to generate BDEC chunks with {@link ArrowBatchWriteMode#STREAM}. */
+    ONE(1),
+
+    /** Uses Arrow to generate BDEC chunks with {@link ArrowBatchWriteMode#FILE}. */
+    TWO(2);
+
+    private final byte version;
+
+    BdecVerion(int version) {
+      if (version > Byte.MAX_VALUE || version < Byte.MIN_VALUE) {
+        throw new IllegalArgumentException("Version does not fit into the byte data type");
+      }
+      this.version = (byte) version;
+    }
+
+    public byte toByte() {
+      return version;
+    }
+
+    public static BdecVerion fromInt(int val) {
+      if (val > Byte.MAX_VALUE || val < Byte.MIN_VALUE) {
+        throw new IllegalArgumentException("Version does not fit into the byte data type");
+      }
+      byte version = (byte) val;
+      for (BdecVerion eversion : BdecVerion.values()) {
+        if (eversion.version == version) {
+          return eversion;
+        }
+      }
+      throw new IllegalArgumentException(
+          String.format(
+              "Unsupported BLOB_FORMAT_VERSION = '%d', allowed values are %s",
+              version, Arrays.asList(BdecVerion.values())));
+    }
+  }
+
   // Parameters
   public static final boolean DISABLE_BACKGROUND_FLUSH = false;
   public static final boolean COMPRESS_BLOB_TWICE = false;
diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
index 01400f118..1882fec04 100644
--- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
+++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -17,6 +17,8 @@ public class ParameterProvider {
   public static final String ENABLE_SNOWPIPE_STREAMING_METRICS_MAP_KEY =
       "ENABLE_SNOWPIPE_STREAMING_JMX_METRICS".toLowerCase();
 
+  public static final String BLOB_FORMAT_VERSION = "BLOB_FORMAT_VERSION".toLowerCase();
+
   // Default values
   public static final long BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT = 1000;
   public static final long BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_DEFAULT = 100;
@@ -24,6 +26,8 @@ public class ParameterProvider {
   public static final long INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_DEFAULT = 10;
   public static final boolean SNOWPIPE_STREAMING_METRICS_DEFAULT = false;
 
+  public static final Constants.BdecVerion BLOB_FORMAT_VERSION_DEFAULT = Constants.BdecVerion.ONE;
+
   /** Map of parameter name to parameter value. This will be set by client/configure API Call. */
   private final Map<String, Object> parameterMap = new HashMap<>();
 
@@ -90,6 +94,9 @@ private void setParameterMap(Map<String, Object> parameterOverrides, Properties
         SNOWPIPE_STREAMING_METRICS_DEFAULT,
         parameterOverrides,
         props);
+
+    this.updateValue(BLOB_FORMAT_VERSION, BLOB_FORMAT_VERSION_DEFAULT, parameterOverrides, props);
+    getBlobFormatVersion(); // to verify parsing the configured value
   }
 
   /** @return Longest interval in milliseconds between buffer flushes */
@@ -130,6 +137,23 @@ public boolean hasEnabledSnowpipeStreamingMetrics() {
             ENABLE_SNOWPIPE_STREAMING_METRICS_MAP_KEY, SNOWPIPE_STREAMING_METRICS_DEFAULT);
   }
 
+  /** @return Blob format version: 1 (arrow stream write mode), 2 (arrow file write mode) etc */
+  public Constants.BdecVerion getBlobFormatVersion() {
+    Object val = this.parameterMap.getOrDefault(BLOB_FORMAT_VERSION, BLOB_FORMAT_VERSION_DEFAULT);
+    if (val instanceof Constants.BdecVerion) {
+      return (Constants.BdecVerion) val;
+    }
+    if (val instanceof String) {
+      try {
+        val = Integer.parseInt((String) val);
+      } catch (Throwable t) {
+        throw new IllegalArgumentException(
+            String.format("Failed to parse BLOB_FORMAT_VERSION = '%s'", val), t);
+      }
+    }
+    return Constants.BdecVerion.fromInt((int) val);
+  }
+
   @Override
   public String toString() {
     return "ParameterProvider{" + "parameterMap=" + parameterMap + '}';
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index ea72b62bf..7f26721fa 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -13,6 +13,8 @@
 import static net.snowflake.ingest.utils.Constants.SSL;
 import static net.snowflake.ingest.utils.Constants.USER;
 import static net.snowflake.ingest.utils.Constants.WAREHOUSE;
+import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION;
+import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -187,9 +189,16 @@ public static Properties getProperties() throws Exception {
     props.put(PRIVATE_KEY, privateKeyPem);
     props.put(ROLE, role);
     props.put(ACCOUNT_URL, getAccountURL());
+    props.put(BLOB_FORMAT_VERSION, getBlobFormatVersion());
     return props;
   }
 
+  private static byte getBlobFormatVersion() {
+    return profile.has(BLOB_FORMAT_VERSION)
+        ? (byte) profile.get(BLOB_FORMAT_VERSION).asInt()
+        : BLOB_FORMAT_VERSION_DEFAULT.toByte();
+  }
+
   /**
    * Create snowflake jdbc connection
    *
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index a14b3c1ae..7e08ca6cf 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -4,7 +4,6 @@
 import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
 import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
 import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_FORMAT_VERSION;
 import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
 import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
 import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
@@ -35,6 +34,7 @@
 import net.snowflake.client.jdbc.SnowflakeConnectionV1;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.Cryptor;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.ParameterProvider;
@@ -516,7 +516,9 @@ public void testBlobBuilder() throws Exception {
 
     chunksMetadataList.add(chunkMetadata);
 
-    byte[] blob = BlobBuilder.build(chunksMetadataList, chunksDataList, checksum, dataSize);
+    final Constants.BdecVerion bdecVersion = ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT;
+    byte[] blob =
+        BlobBuilder.build(chunksMetadataList, chunksDataList, checksum, dataSize, bdecVersion);
 
     // Read the blob byte array back to valid the behavior
     InputStream input = new ByteArrayInputStream(blob);
@@ -528,8 +530,7 @@ public void testBlobBuilder() throws Exception {
               Arrays.copyOfRange(blob, offset, offset += BLOB_TAG_SIZE_IN_BYTES),
               StandardCharsets.UTF_8));
       Assert.assertEquals(
-          BLOB_FORMAT_VERSION,
-          Arrays.copyOfRange(blob, offset, offset += BLOB_VERSION_SIZE_IN_BYTES)[0]);
+          bdecVersion, Arrays.copyOfRange(blob, offset, offset += BLOB_VERSION_SIZE_IN_BYTES)[0]);
       long totalSize =
           ByteBuffer.wrap(Arrays.copyOfRange(blob, offset, offset += BLOB_FILE_SIZE_SIZE_IN_BYTES))
               .getLong();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index 0b4f1b3a5..c9d754632 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -618,6 +618,7 @@ public void testGetRetryBlobs() throws Exception {
         result.get(0).getChunks().get(0).getChannels().stream()
             .map(c -> c.getChannelName())
             .collect(Collectors.toSet()));
+    Assert.assertEquals(ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT, result.get(0).getVersion());
   }
 
   @Test
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 3c0d7c4fb..233bca1bc 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -3,6 +3,7 @@
 import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
 import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
 
+import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.sql.Connection;
 import java.sql.ResultSet;
@@ -14,6 +15,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Random;
 import java.util.TimeZone;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
@@ -42,6 +44,7 @@ public class StreamingIngestIT {
   private static final String TEST_SCHEMA = "STREAMING_INGEST_TEST_SCHEMA";
 
   private static final String INTERLEAVED_TABLE_PREFIX = "t_interleaved_test_";
+  private static final String INTERLEAVED_CHANNEL_TABLE = "t_interleaved_channel_test";
   private static final int INTERLEAVED_CHANNEL_NUMBER = 3;
 
   private Properties prop;
@@ -220,7 +223,7 @@ public void testInterleavedIngest() {
 
     SnowflakeStreamingIngestChannel[] channels =
         new SnowflakeStreamingIngestChannel[INTERLEAVED_CHANNEL_NUMBER];
-    iter.accept(i -> channels[i - 1] = openChannel(INTERLEAVED_TABLE_PREFIX + i));
+    iter.accept(i -> channels[i - 1] = openChannel(INTERLEAVED_TABLE_PREFIX + i, "CHANNEL"));
 
     iter.accept(
         i ->
@@ -228,7 +231,40 @@ public void testInterleavedIngest() {
                 channels[i - 1], INTERLEAVED_TABLE_PREFIX + i, 1 << (i + 1)));
     iter.accept(i -> waitChannelFlushed(channels[i - 1], 1 << (i + 1)));
 
-    iter.accept(i -> verifyInterleavedResult(1 << (i + 1), INTERLEAVED_TABLE_PREFIX + i));
+    iter.accept(i -> verifyTableRowCount(1 << (i + 1), INTERLEAVED_TABLE_PREFIX + i));
+    iter.accept(
+        i ->
+            verifyInterleavedResult(
+                1 << (i + 1), INTERLEAVED_TABLE_PREFIX + i, INTERLEAVED_TABLE_PREFIX + i));
+  }
+
+  @Test
+  public void testMultiChannelChunk() {
+    Consumer<IntConsumer> iter =
+        f -> IntStream.rangeClosed(1, INTERLEAVED_CHANNEL_NUMBER).forEach(f);
+
+    createTableForInterleavedTest(INTERLEAVED_CHANNEL_TABLE);
+
+    SnowflakeStreamingIngestChannel[] channels =
+        new SnowflakeStreamingIngestChannel[INTERLEAVED_CHANNEL_NUMBER];
+    iter.accept(i -> channels[i - 1] = openChannel(INTERLEAVED_CHANNEL_TABLE, "CHANNEL_" + i));
+
+    iter.accept(
+        i ->
+            produceRowsForInterleavedTest(
+                channels[i - 1], INTERLEAVED_CHANNEL_TABLE + "_channel_" + i, 1 << (i + 1)));
+    iter.accept(i -> waitChannelFlushed(channels[i - 1], 1 << (i + 1)));
+
+    int rowNumber =
+        IntStream.rangeClosed(1, INTERLEAVED_CHANNEL_NUMBER).map(i -> 1 << (i + 1)).sum();
+    verifyTableRowCount(rowNumber, INTERLEAVED_CHANNEL_TABLE);
+
+    iter.accept(
+        i ->
+            verifyInterleavedResult(
+                1 << (i + 1),
+                INTERLEAVED_CHANNEL_TABLE,
+                INTERLEAVED_CHANNEL_TABLE + "_channel_" + i));
   }
 
   @Test
@@ -936,9 +972,9 @@ private void createTableForInterleavedTest(String tableName) {
     }
   }
 
-  private SnowflakeStreamingIngestChannel openChannel(String tableName) {
+  private SnowflakeStreamingIngestChannel openChannel(String tableName, String channelName) {
     OpenChannelRequest request =
-        OpenChannelRequest.builder("CHANNEL")
+        OpenChannelRequest.builder(channelName)
             .setDBName(TEST_DB)
             .setSchemaName(TEST_SCHEMA)
             .setTableName(tableName)
@@ -950,20 +986,22 @@ private SnowflakeStreamingIngestChannel openChannel(String tableName) {
   }
 
   private void produceRowsForInterleavedTest(
-      SnowflakeStreamingIngestChannel channel, String tableName, int rowNumber) {
+      SnowflakeStreamingIngestChannel channel, String strPrefix, int rowNumber) {
     // Insert a few rows into the channel
     for (int val = 0; val < rowNumber; val++) {
       Map<String, Object> row = new HashMap<>();
       row.put("num", val);
-      row.put("str", tableName + "_value_" + val);
+      row.put("str", strPrefix + "_value_" + val);
       verifyInsertValidationResponse(channel.insertRow(row, Integer.toString(val)));
     }
   }
 
   private void waitChannelFlushed(SnowflakeStreamingIngestChannel channel, int numberOfRows) {
+    String latestCommittedOffsetToken = null;
     for (int i = 1; i < 15; i++) {
-      if (channel.getLatestCommittedOffsetToken() != null
-          && channel.getLatestCommittedOffsetToken().equals(Integer.toString(numberOfRows - 1))) {
+      latestCommittedOffsetToken = channel.getLatestCommittedOffsetToken();
+      if (latestCommittedOffsetToken != null
+          && latestCommittedOffsetToken.equals(Integer.toString(numberOfRows - 1))) {
         return;
       }
       try {
@@ -973,25 +1011,163 @@ private void waitChannelFlushed(SnowflakeStreamingIngestChannel channel, int num
             "Interrupted waitChannelFlushed for " + numberOfRows + " rows", e);
       }
     }
-    Assert.fail("Row sequencer not updated before timeout");
+    Assert.fail(
+        "Row sequencer not updated before timeout, latestCommittedOffsetToken: "
+            + latestCommittedOffsetToken);
   }
 
-  private void verifyInterleavedResult(int rowNumber, String tableName) {
-    ResultSet result;
+  private void verifyInterleavedResult(int rowNumber, String tableName, String strPrefix) {
     try {
-      result =
+      ResultSet result =
           jdbcConnection
               .createStatement()
               .executeQuery(
                   String.format(
-                      "select * from %s.%s.%s order by num", TEST_DB, TEST_SCHEMA, tableName));
+                      "select * from %s.%s.%s where str ilike '%s%%' order by num",
+                      TEST_DB, TEST_SCHEMA, tableName, strPrefix));
       for (int val = 0; val < rowNumber; val++) {
         result.next();
         Assert.assertEquals(val, result.getLong("NUM"));
-        Assert.assertEquals(tableName + "_value_" + val, result.getString("STR"));
+        Assert.assertEquals(strPrefix + "_value_" + val, result.getString("STR"));
       }
     } catch (SQLException e) {
       throw new RuntimeException("Cannot verifyInterleavedResult for " + tableName, e);
     }
   }
+
+  private void verifyTableRowCount(int rowNumber, String tableName) {
+    try {
+      ResultSet resultCount =
+          jdbcConnection
+              .createStatement()
+              .executeQuery(
+                  String.format("select count(*) from %s.%s.%s", TEST_DB, TEST_SCHEMA, tableName));
+      resultCount.next();
+      Assert.assertEquals(rowNumber, resultCount.getLong(1));
+    } catch (SQLException e) {
+      throw new RuntimeException("Cannot verifyTableRowCount for " + tableName, e);
+    }
+  }
+
+  @Test
+  public void testDataTypes() {
+    String tableName = "t_data_types";
+    try {
+      jdbcConnection
+          .createStatement()
+          .execute(
+              String.format(
+                  "create or replace table %s (num NUMBER, -- default FIXED\n"
+                      + "                                    num_10_5 NUMBER(10, 5), -- FIXED with"
+                      + " non zero scale: big dec\n"
+                      + "                                    num_38_0 NUMBER(38, 0), -- FIXED sb16:"
+                      + " big dec\n"
+                      + "                                    num_2_0 NUMBER(2, 0), -- FIXED sb1\n"
+                      + "                                    num_4_0 NUMBER(4, 0), -- FIXED sb2\n"
+                      + "                                    num_9_0 NUMBER(9, 0), -- FIXED sb4\n"
+                      + "                                    num_18_0 NUMBER(18, 0), -- FIXED sb8\n"
+                      + "                                    num_float FLOAT, -- REAL\n"
+                      + "                                    str_varchar VARCHAR(256), -- varchar,"
+                      + " char, any, string, text\n"
+                      + "                                    bin BINARY(256), -- BINARY, VARBINARY:"
+                      + " hex\n"
+                      + "                                    bl BOOLEAN, -- BOOLEAN: true/false\n"
+                      + "                                    var VARIANT, -- VARIANT: json\n"
+                      + "                                    obj OBJECT, -- OBJECT: json map\n"
+                      + "                                    arr ARRAY, -- ARRAY: array of"
+                      + " variants\n"
+                      + "                                    epochdays DATE,\n"
+                      + "                                    timesec TIME(0),\n"
+                      + "                                    timenano TIME(9),\n"
+                      + "                                    epochsec TIMESTAMP_NTZ(0),\n"
+                      + "                                    epochnano TIMESTAMP_NTZ(9));",
+                  tableName));
+    } catch (SQLException e) {
+      throw new RuntimeException("Cannot create table " + tableName, e);
+    }
+
+    SnowflakeStreamingIngestChannel channel = openChannel(tableName, "CHANNEL");
+
+    Random r = new Random();
+    for (int val = 0; val < 10000; val++) {
+      verifyInsertValidationResponse(channel.insertRow(getRandomRow(r), Integer.toString(val)));
+    }
+    waitChannelFlushed(channel, 10000);
+  }
+
+  private static Map<String, Object> getRandomRow(Random r) {
+    Map<String, Object> row = new HashMap<>();
+    row.put("num", r.nextInt());
+    row.put("num_10_5", nextFloat(r));
+    row.put(
+        "num_38_0",
+        new BigDecimal("" + nextLongOfPrecision(r, 18) + Math.abs(nextLongOfPrecision(r, 18))));
+    row.put("num_2_0", r.nextInt(100));
+    row.put("num_4_0", r.nextInt(10000));
+    row.put("num_9_0", r.nextInt(1000000000));
+    row.put("num_18_0", nextLongOfPrecision(r, 18));
+    row.put("num_float", nextFloat(r));
+    row.put("str_varchar", nextString(r));
+    row.put("bin", nextBytes(r));
+    row.put("bl", r.nextBoolean());
+    row.put("var", nextJson(r));
+    row.put("obj", nextJson(r));
+    row.put("arr", Arrays.asList(r.nextInt(100), r.nextInt(100), r.nextInt(100)));
+    row.put("epochdays", Math.abs(r.nextInt()) % 18963); // DATE, 02.12.2021
+    row.put(
+        "timesec",
+        (r.nextInt(11) * 60 * 60 + r.nextInt(59) * 60 + r.nextInt(59)) * 10000
+            + r.nextInt(9999)); // TIME(4), 05:12:43.4536
+    row.put(
+        "timenano",
+        (14 * 60 * 60 + 26 * 60 + 34) * 1000000000L + 437582643); // TIME(9), 14:26:34.437582643
+    row.put(
+        "epochsec", Math.abs(r.nextLong()) % 1638459438); // TIMESTAMP_LTZ(0), 02.12.2021 15:37:18
+    row.put(
+        "epochnano",
+        new BigDecimal(
+            (Math.abs(r.nextInt()) % 1999999999)
+                + "."
+                + Math.abs(
+                    r.nextInt(999999999)))); // TIMESTAMP_LTZ(9), 18.05.2033 03:33:19.999999999
+    return row;
+  }
+
+  private static long nextLongOfPrecision(Random r, int precision) {
+    return r.nextLong() % Math.round(Math.pow(10, precision));
+  }
+
+  private static String nextString(Random r) {
+    return new String(nextBytes(r));
+  }
+
+  private static byte[] nextBytes(Random r) {
+    byte[] bin = new byte[128];
+    r.nextBytes(bin);
+    for (int i = 0; i < bin.length; i++) {
+      bin[i] = (byte) (Math.abs(bin[i]) % 25 + 97); // ascii letters
+    }
+    return bin;
+  }
+
+  private static double nextFloat(Random r) {
+    return (r.nextLong() % Math.round(Math.pow(10, 10))) / 100000d;
+  }
+
+  private static String nextJson(Random r) {
+    return String.format(
+        "{ \"%s\": %d, \"%s\": \"%s\", \"%s\": null, \"%s\": { \"%s\": %f, \"%s\": \"%s\", \"%s\":"
+            + " null } }",
+        nextString(r),
+        r.nextInt(),
+        nextString(r),
+        nextString(r),
+        nextString(r),
+        nextString(r),
+        nextString(r),
+        r.nextFloat(),
+        nextString(r),
+        nextString(r),
+        nextString(r));
+  }
 }

From 9895e208a837720fda59922731d6b4a00a916b1c Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 20 Jul 2022 23:30:10 -0700
Subject: [PATCH 047/356] SNOW-356156: support client telemetry through
 client_telemetry_v (#195)

This PR contains the following major changes:

1. Add the ability in the ingest SDK to upload telemetries through the sessionless client telemetry framework
2. Add a few client side metrics for Streaming Ingest and upload them to SF
3. Add corresponding unit test, IT test should already covered by existing tests
---
 pom.xml                                       |  29 +--
 .../snowflake/ingest/SimpleIngestManager.java |   8 +-
 .../ingest/connection/RequestBuilder.java     | 113 ++++++++---
 .../ingest/connection/SecurityManager.java    |  46 +++--
 .../connection/ServiceResponseHandler.java    |  10 +-
 .../ingest/connection/TelemetryService.java   | 143 ++++++++++++++
 .../streaming/internal/FlushService.java      |  17 +-
 .../streaming/internal/RegisterService.java   |  32 ++-
 ...nowflakeStreamingIngestClientInternal.java | 181 +++++++++++------
 .../internal/StreamingIngestStage.java        |   4 +-
 .../internal/StreamingIngestUtils.java        |   4 +-
 .../net/snowflake/ingest/utils/Constants.java |   4 +-
 .../net/snowflake/ingest/utils/HttpUtil.java  |  54 +++--
 .../net/snowflake/ingest/SimpleIngestIT.java  |   6 +-
 .../connection/SecurityManagerTest.java       | 186 ------------------
 .../connection/TelemetryServiceTest.java      |  76 +++++++
 .../ingest/connection/TestKeyRenewal.java     |   3 +-
 .../SnowflakeStreamingIngestChannelTest.java  |  15 +-
 .../SnowflakeStreamingIngestClientTest.java   |  15 +-
 .../internal/StreamingIngestStageTest.java    |   8 +-
 .../internal/StreamingIngestUtilsTest.java    |   8 +-
 21 files changed, 569 insertions(+), 393 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/connection/TelemetryService.java
 delete mode 100644 src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java
 create mode 100644 src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java

diff --git a/pom.xml b/pom.xml
index d2599b287..52a35f01b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -198,7 +198,7 @@
         <dependency>
             <groupId>net.snowflake</groupId>
             <artifactId>snowflake-jdbc</artifactId>
-            <version>3.13.14</version>
+            <version>3.13.15</version>
         </dependency>
 
         <!-- String collation-->
@@ -229,33 +229,6 @@
             <version>2.13.2.1</version>
         </dependency>
 
-        <!-- Apache HTTP Components for actually sending requests over the network -->
-        <dependency>
-            <groupId>org.apache.httpcomponents</groupId>
-            <artifactId>httpclient</artifactId>
-            <version>4.5.13</version>
-            <exclusions>
-                <exclusion>  <!-- declare the exclusion here -->
-                    <groupId>commons-codec</groupId>
-                    <artifactId>commons-codec</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-
-        <dependency>
-            <groupId>commons-codec</groupId>
-            <artifactId>commons-codec</artifactId>
-            <version>1.15</version>
-        </dependency>
-
-
-        <!-- the Async HTTP Client so we can delay execution -->
-        <dependency>
-            <groupId>org.apache.httpcomponents</groupId>
-            <artifactId>httpasyncclient</artifactId>
-            <version>4.1.2</version>
-        </dependency>
-
 
         <!-- SLF4J api that a client can shim in later -->
         <dependency>
diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
index 756b5df9c..e5163a4dc 100644
--- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
+++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
@@ -19,6 +19,10 @@
 import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpGet;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.ingest.connection.ClientStatusResponse;
 import net.snowflake.ingest.connection.ConfigureClientResponse;
 import net.snowflake.ingest.connection.HistoryRangeResponse;
@@ -32,10 +36,6 @@
 import net.snowflake.ingest.utils.HttpUtil;
 import net.snowflake.ingest.utils.StagedFileWrapper;
 import net.snowflake.ingest.utils.Utils;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.impl.client.CloseableHttpClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index f63029431..0ff50f6d5 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -4,6 +4,7 @@
 
 package net.snowflake.ingest.connection;
 
+import static net.snowflake.ingest.utils.Constants.ENABLE_TELEMETRY_TO_SF;
 import static net.snowflake.ingest.utils.Utils.isNullOrEmpty;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
@@ -19,15 +20,19 @@
 import java.util.Optional;
 import java.util.Properties;
 import java.util.UUID;
+import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpGet;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest;
+import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder;
+import net.snowflake.client.jdbc.internal.apache.http.entity.ContentType;
+import net.snowflake.client.jdbc.internal.apache.http.entity.StringEntity;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.ingest.SimpleIngestManager;
-import net.snowflake.ingest.utils.*;
-import org.apache.http.HttpHeaders;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.SnowflakeURL;
+import net.snowflake.ingest.utils.StagedFileWrapper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -44,7 +49,7 @@ public class RequestBuilder {
   /* Member variables Begin */
 
   // the security manager which will handle token generation
-  private SecurityManager securityManager;
+  private final SecurityManager securityManager;
 
   // whatever the actual scheme is
   private String scheme;
@@ -57,6 +62,9 @@ public class RequestBuilder {
 
   private final String userAgentSuffix;
 
+  // Reference to the telemetry service
+  private final TelemetryService telemetryService;
+
   /* Member variables End */
 
   /* Static constants Begin */
@@ -149,6 +157,7 @@ public RequestBuilder(String accountName, String userName, KeyPair keyPair) {
    * @param accountName - the name of the Snowflake account to which we're connecting
    * @param userName - the username of the entity loading files
    * @param keyPair - the Public/Private key pair we'll use to authenticate
+   * @param userAgentSuffix - The suffix part of HTTP Header User-Agent
    */
   public RequestBuilder(
       String accountName,
@@ -199,13 +208,74 @@ public RequestBuilder(
       String hostName,
       int portNum,
       String userAgentSuffix) {
+    this(
+        accountName, userName, keyPair, schemeName, hostName, portNum, userAgentSuffix, null, null);
+  }
+
+  /**
+   * RequestBuilder - constructor used by streaming ingest
+   *
+   * @param url - the Snowflake account to which we're connecting
+   * @param userName - the username of the entity loading files
+   * @param keyPair - the Public/Private key pair we'll use to authenticate
+   * @param httpClient - reference to the http client
+   * @param clientName - name of the client, used to uniquely identify a client if used
+   */
+  public RequestBuilder(
+      SnowflakeURL url,
+      String userName,
+      KeyPair keyPair,
+      CloseableHttpClient httpClient,
+      String clientName) {
+    this(
+        url.getAccount(),
+        userName,
+        keyPair,
+        url.getScheme(),
+        url.getUrlWithoutPort(),
+        url.getPort(),
+        null,
+        httpClient,
+        clientName);
+  }
+
+  /**
+   * RequestBuilder - this constructor is for testing purposes only
+   *
+   * @param accountName - the account name to which we're connecting
+   * @param userName - for whom are we connecting?
+   * @param keyPair - our auth credentials
+   * @param schemeName - are we HTTP or HTTPS?
+   * @param hostName - the host for this snowflake instance
+   * @param portNum - the port number
+   * @param userAgentSuffix - The suffix part of HTTP Header User-Agent
+   * @param httpClient - reference to the http client
+   * @param clientName - name of the client, used to uniquely identify a client if used
+   */
+  public RequestBuilder(
+      String accountName,
+      String userName,
+      KeyPair keyPair,
+      String schemeName,
+      String hostName,
+      int portNum,
+      String userAgentSuffix,
+      CloseableHttpClient httpClient,
+      String clientName) {
     // none of these arguments should be null
     if (accountName == null || userName == null || keyPair == null) {
       throw new IllegalArgumentException();
     }
 
+    // Set up the telemetry service if needed
+    this.telemetryService =
+        ENABLE_TELEMETRY_TO_SF
+            ? new TelemetryService(
+                httpClient, clientName, schemeName + "://" + hostName + ":" + portNum)
+            : null;
+
     // create our security/token manager
-    securityManager = new SecurityManager(accountName, userName, keyPair);
+    securityManager = new SecurityManager(accountName, userName, keyPair, telemetryService);
 
     // stash references to the account and user name as well
     String account = accountName.toUpperCase();
@@ -228,23 +298,6 @@ public RequestBuilder(
         this.userAgentSuffix);
   }
 
-  /**
-   * RequestBuilder - constructor used by streaming ingest
-   *
-   * @param url - the Snowflake account to which we're connecting
-   * @param userName - the username of the entity loading files
-   * @param keyPair - the Public/Private key pair we'll use to authenticate
-   */
-  public RequestBuilder(SnowflakeURL url, String userName, KeyPair keyPair) {
-    this(
-        url.getAccount(),
-        userName,
-        keyPair,
-        url.getScheme(),
-        url.getUrlWithoutPort(),
-        url.getPort());
-  }
-
   private static Properties loadProperties() {
     Properties properties = new Properties();
     properties.put("version", DEFAULT_VERSION);
@@ -804,5 +857,11 @@ public HttpGet generateGetClientStatusRequest(UUID requestID, String pipe)
    */
   public void closeResources() {
     securityManager.close();
+    telemetryService.close();
+  }
+
+  /** Get the telemetry service */
+  public TelemetryService getTelemetryService() {
+    return telemetryService;
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/connection/SecurityManager.java b/src/main/java/net/snowflake/ingest/connection/SecurityManager.java
index 7832ab88e..581418479 100644
--- a/src/main/java/net/snowflake/ingest/connection/SecurityManager.java
+++ b/src/main/java/net/snowflake/ingest/connection/SecurityManager.java
@@ -41,22 +41,22 @@ final class SecurityManager implements AutoCloseable {
   private static final int RENEWAL_INTERVAL = 54;
 
   // The public - private key pair we're using to connect to the service
-  private transient KeyPair keyPair;
+  private final transient KeyPair keyPair;
 
   // the name of the account on behalf of which we're connecting
-  private String account;
+  private final String account;
 
   // Fingerprint of public key sent from client in jwt payload
   private String publicKeyFingerPrint;
 
   // the name of the user who will be loading the files
-  private String user;
+  private final String user;
 
   // the token itself
-  private AtomicReference<String> token;
+  private final AtomicReference<String> token;
 
   // Did we fail to regenerate our token at some point?
-  private AtomicBoolean regenFailed;
+  private final AtomicBoolean regenFailed;
 
   // Thread factory for daemon threads so that application can shutdown
   final ThreadFactory tf = ThreadFactoryUtil.poolThreadFactory(getClass().getSimpleName(), true);
@@ -64,23 +64,32 @@ final class SecurityManager implements AutoCloseable {
   // the thread we use for renewing all tokens
   private final ScheduledExecutorService keyRenewer = Executors.newScheduledThreadPool(1, tf);
 
+  // Reference to the Telemetry Service in the client
+  private final TelemetryService telemetryService;
+
   /**
    * Creates a SecurityManager entity for a given account, user and KeyPair with a specified time to
    * renew the token
    *
-   * @param accountname - the snowflake account name of this user
+   * @param accountName - the snowflake account name of this user
    * @param username - the snowflake username of the current user
    * @param keyPair - the public/private key pair we're using to connect
    * @param timeTillRenewal - the time measure until we renew the token
    * @param unit the unit by which timeTillRenewal is measured
+   * @param telemetryService reference to the telemetry service
    */
   SecurityManager(
-      String accountname, String username, KeyPair keyPair, int timeTillRenewal, TimeUnit unit) {
+      String accountName,
+      String username,
+      KeyPair keyPair,
+      int timeTillRenewal,
+      TimeUnit unit,
+      TelemetryService telemetryService) {
     // if any of our arguments are null, throw an exception
-    if (accountname == null || username == null || keyPair == null) {
+    if (accountName == null || username == null || keyPair == null) {
       throw new IllegalArgumentException();
     }
-    account = parseAccount(accountname);
+    account = parseAccount(accountName);
     user = username.toUpperCase();
 
     // create our automatic reference to a string (our token)
@@ -92,6 +101,8 @@ final class SecurityManager implements AutoCloseable {
     // we have to keep around the keys
     this.keyPair = keyPair;
 
+    this.telemetryService = telemetryService;
+
     // generate our first token
     regenerateToken();
 
@@ -101,20 +112,22 @@ final class SecurityManager implements AutoCloseable {
 
   /**
    * Creates a SecurityManager entity for a given account, user and KeyPair with the default time to
-   * renew (54 minutes)
+   * renew (RENEWAL_INTERVAL minutes)
    *
-   * @param accountname - the snowflake account name of this user
+   * @param accountName - the snowflake account name of this user
    * @param username - the snowflake username of the current user
    * @param keyPair - the public/private key pair we're using to connect
+   * @param telemetryService reference to the telemetry service
    */
-  SecurityManager(String accountname, String username, KeyPair keyPair) {
-    this(accountname, username, keyPair, RENEWAL_INTERVAL, TimeUnit.MINUTES);
+  SecurityManager(
+      String accountName, String username, KeyPair keyPair, TelemetryService telemetryService) {
+    this(accountName, username, keyPair, RENEWAL_INTERVAL, TimeUnit.MINUTES, telemetryService);
   }
 
   /**
    * Trims an account name if it contains a <b>"."</b>
    *
-   * <p>Snowflake's python connector trims an accountname if it contains a "."
+   * <p>Snowflake's python connector trims an accountName if it contains a "."
    *
    * @param accountName given accountName in SimpleIngestManager's constructor
    * @return initial part of account name if originally it contained a ".", otherwise return the
@@ -177,6 +190,11 @@ private void regenerateToken() {
     // atomically update the string
     LOGGER.info("Created new JWT");
     token.set(newToken);
+
+    // Refresh the token used in the telemetry service as well
+    if (telemetryService != null) {
+      telemetryService.refreshJWTToken(newToken);
+    }
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java
index ea3ae715e..f3da23991 100644
--- a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java
+++ b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java
@@ -8,12 +8,12 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
 import java.util.UUID;
+import net.snowflake.client.jdbc.internal.apache.http.HttpEntity;
+import net.snowflake.client.jdbc.internal.apache.http.HttpResponse;
+import net.snowflake.client.jdbc.internal.apache.http.HttpStatus;
+import net.snowflake.client.jdbc.internal.apache.http.StatusLine;
+import net.snowflake.client.jdbc.internal.apache.http.util.EntityUtils;
 import net.snowflake.ingest.utils.BackOffException;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.StatusLine;
-import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java
new file mode 100644
index 000000000..31105c189
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.connection;
+
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Snapshot;
+import com.codahale.metrics.Timer;
+import java.util.concurrent.TimeUnit;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
+import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
+import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
+import net.snowflake.client.jdbc.telemetry.TelemetryClient;
+import net.snowflake.client.jdbc.telemetry.TelemetryUtil;
+import net.snowflake.ingest.utils.Logging;
+
+/**
+ * Telemetry service to collect logs in the SDK and send them to Snowflake through the JDBC client
+ * telemetry API
+ */
+public class TelemetryService {
+  // Enum for different client telemetries
+  enum TelemetryType {
+    STREAMING_INGEST_LATENCY_IN_SEC("streaming_ingest_latency_in_ms"),
+    STREAMING_INGEST_CLIENT_FAILURE("streaming_ingest_client_failure"),
+    STREAMING_INGEST_THROUGHPUT_BYTES_PER_SEC("streaming_ingest_throughput_bytes_per_sec"),
+    STREAMING_INGEST_CPU_MEMORY_USAGE("streaming_ingest_cpu_memory_usage");
+
+    private final String name;
+
+    TelemetryType(String name) {
+      this.name = name;
+    }
+
+    @Override
+    public String toString() {
+      return this.name;
+    }
+  }
+
+  private static final Logging logger = new Logging(TelemetryService.class);
+  private static final ObjectMapper MAPPER = new ObjectMapper();
+
+  private static final String TYPE = "type";
+  private static final String CLIENT_NAME = "client_name";
+  private static final String COUNT = "count";
+  private static final String MAX = "max";
+  private static final String MIN = "min";
+  private static final String MEDIAN = "median";
+  private static final String MEAN = "mean";
+  private static final String PERCENTILE99TH = "99thPercentile";
+  private final TelemetryClient telemetry;
+  private final String clientName;
+
+  /**
+   * Default constructor
+   *
+   * @param httpClient http client
+   * @param clientName name of the client
+   * @param url account url
+   */
+  TelemetryService(CloseableHttpClient httpClient, String clientName, String url) {
+    this.clientName = clientName;
+    this.telemetry = (TelemetryClient) TelemetryClient.createSessionlessTelemetry(httpClient, url);
+  }
+
+  /** Flush the telemetry buffer and close the telemetry service */
+  public void close() {
+    this.telemetry.close();
+  }
+
+  /** Report the Streaming Ingest latency metrics */
+  public void reportLatencyInSec(
+      Timer buildLatency, Timer uploadLatency, Timer registerLatency, Timer flushLatency) {
+    ObjectNode msg = MAPPER.createObjectNode();
+    msg.set("build_latency_sec", buildMsgFromTimer(buildLatency));
+    msg.set("upload_latency_sec", buildMsgFromTimer(uploadLatency));
+    msg.set("register_latency_sec", buildMsgFromTimer(registerLatency));
+    msg.set("flush_latency_sec", buildMsgFromTimer(flushLatency));
+    send(TelemetryType.STREAMING_INGEST_LATENCY_IN_SEC, msg);
+  }
+
+  /** Report the Streaming Ingest failure metrics */
+  public void reportClientFailure(String summary, String exception) {
+    ObjectNode msg = MAPPER.createObjectNode();
+    msg.put("summary", summary);
+    msg.put("exception", exception);
+    send(TelemetryType.STREAMING_INGEST_CLIENT_FAILURE, msg);
+  }
+
+  /** Report the Streaming Ingest throughput metrics */
+  public void reportThroughputBytesPerSecond(Meter inputThroughput, Meter uploadThrough) {
+    ObjectNode msg = MAPPER.createObjectNode();
+    msg.put("input_mean_rate_bytes_per_sec", inputThroughput.getMeanRate());
+    msg.put("upload_mean_rate_bytes_per_sec", uploadThrough.getMeanRate());
+    send(TelemetryType.STREAMING_INGEST_THROUGHPUT_BYTES_PER_SEC, msg);
+  }
+
+  /** Report the Streaming Ingest CUP/memory usage metrics */
+  public void reportCpuMemoryUsage(Histogram cpuUsage) {
+    ObjectNode msg = MAPPER.createObjectNode();
+    Snapshot cpuSnapshot = cpuUsage.getSnapshot();
+    Runtime runTime = Runtime.getRuntime();
+    msg.put("cpu_max", cpuSnapshot.getMax());
+    msg.put("cpu_mean", cpuSnapshot.getMean());
+    msg.put("max_memory", runTime.maxMemory());
+    msg.put("total_memory", runTime.totalMemory());
+    msg.put("free_memory", runTime.freeMemory());
+    send(TelemetryType.STREAMING_INGEST_CPU_MEMORY_USAGE, msg);
+  }
+
+  /** Send log to Snowflake asynchronously through JDBC client telemetry */
+  void send(TelemetryType type, ObjectNode msg) {
+    try {
+      msg.put(TYPE, type.toString());
+      msg.put(CLIENT_NAME, clientName);
+      telemetry.addLogToBatch(TelemetryUtil.buildJobData(msg));
+    } catch (Exception e) {
+      logger.logWarn("Failed to send telemetry data, error: {}", e.getMessage());
+    }
+  }
+
+  /** Build message from a Timer metric */
+  private ObjectNode buildMsgFromTimer(Timer timer) {
+    ObjectNode msg = MAPPER.createObjectNode();
+    Snapshot buildSnapshot = timer.getSnapshot();
+    msg.put(COUNT, timer.getCount());
+    msg.put(MAX, TimeUnit.NANOSECONDS.toMillis(buildSnapshot.getMax()));
+    msg.put(MIN, TimeUnit.NANOSECONDS.toMillis(buildSnapshot.getMin()));
+    msg.put(MEAN, TimeUnit.NANOSECONDS.toMillis((long) buildSnapshot.getMean()));
+    msg.put(MEDIAN, TimeUnit.NANOSECONDS.toMillis((long) buildSnapshot.getMedian()));
+    msg.put(
+        PERCENTILE99TH, TimeUnit.NANOSECONDS.toMillis((long) buildSnapshot.get99thPercentile()));
+    return msg;
+  }
+
+  /** Refresh JWT token stored in the telemetry client */
+  public void refreshJWTToken(String token) {
+    telemetry.refreshToken(token);
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 4adbcd764..4f1974e8a 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -362,12 +362,17 @@ void distributeFlushTasks() {
                             "buildUploadWorkers stats={}", this.buildUploadWorkers.toString());
                         return buildAndUpload(filePath, blobData);
                       } catch (IOException e) {
-                        logger.logError(
-                            "Building blob failed={}, exception={}, detail={}, all channels in the"
-                                + " blob will be invalidated",
-                            filePath,
-                            e,
-                            e.getMessage());
+                        String errorMessage =
+                            String.format(
+                                "Building blob failed, client=%s, file=%s, exception=%s,"
+                                    + " detail=%s, all channels in the blob will be invalidated",
+                                this.owningClient.getName(), filePath, e, e.getMessage());
+                        logger.logError(errorMessage);
+                        if (this.owningClient.getTelemetryService() != null) {
+                          this.owningClient
+                              .getTelemetryService()
+                              .reportClientFailure(this.getClass().getSimpleName(), errorMessage);
+                        }
                         invalidateAllChannelsInBlob(blobData);
                         return null;
                       } catch (NoSuchAlgorithmException e) {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index f2eca8c8f..216bf3caa 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -141,14 +141,30 @@ List<FlushService.BlobData> registerBlobs(Map<String, Timer.Context> latencyTime
                 retry++;
                 break;
               }
-              logger.logError(
-                  "Building or uploading blob={} failed or timed out, exception={}, detail={},"
-                      + " cause={}, detail={}, all channels in the blob will be invalidated",
-                  futureBlob.getKey().getFilePath(),
-                  e,
-                  e.getMessage(),
-                  e.getCause(),
-                  e.getCause() == null ? null : e.getCause().getMessage());
+              StringBuilder stackTrace = new StringBuilder();
+              if (e.getCause() != null) {
+                for (StackTraceElement element : e.getCause().getStackTrace()) {
+                  stackTrace.append(System.lineSeparator()).append(element.toString());
+                }
+              }
+              String errorMessage =
+                  String.format(
+                      "Building or uploading blob failed, client=%s, file=%s, exception=%s,"
+                          + " detail=%s, cause=%s, detail=%s trace=%s all channels in the blob"
+                          + " will be invalidated",
+                      this.owningClient.getName(),
+                      futureBlob.getKey().getFilePath(),
+                      e,
+                      e.getMessage(),
+                      e.getCause(),
+                      e.getCause() == null ? null : e.getCause().getMessage(),
+                      stackTrace);
+              logger.logError(errorMessage);
+              if (this.owningClient.getTelemetryService() != null) {
+                this.owningClient
+                    .getTelemetryService()
+                    .reportClientFailure(this.getClass().getSimpleName(), errorMessage);
+              }
               this.owningClient
                   .getFlushService()
                   .invalidateAllChannelsInBlob(futureBlob.getKey().getData());
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index aec8311ce..f1e55d1dd 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -12,6 +12,7 @@
 import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.COMMIT_MAX_RETRY_COUNT;
 import static net.snowflake.ingest.utils.Constants.COMMIT_RETRY_INTERVAL_IN_MS;
+import static net.snowflake.ingest.utils.Constants.ENABLE_TELEMETRY_TO_SF;
 import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
 import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
 import static net.snowflake.ingest.utils.Constants.OPEN_CHANNEL_ENDPOINT;
@@ -22,6 +23,7 @@
 import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JMX_METRIC_PREFIX;
 import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JVM_MEMORY_AND_THREAD_METRICS_REGISTRY;
 import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_SHARED_METRICS_REGISTRY;
+import static net.snowflake.ingest.utils.Constants.STREAMING_INGEST_TELEMETRY_UPLOAD_INTERVAL_IN_SEC;
 import static net.snowflake.ingest.utils.Constants.USER;
 
 import com.codahale.metrics.Histogram;
@@ -49,13 +51,17 @@
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 import javax.management.MalformedObjectNameException;
 import javax.management.ObjectName;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.ingest.connection.IngestResponseException;
 import net.snowflake.ingest.connection.RequestBuilder;
+import net.snowflake.ingest.connection.TelemetryService;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.utils.Constants;
@@ -69,7 +75,6 @@
 import net.snowflake.ingest.utils.Utils;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.http.impl.client.CloseableHttpClient;
 
 /**
  * The first version of implementation for SnowflakeStreamingIngestClient. The client internally
@@ -96,7 +101,7 @@ public class SnowflakeStreamingIngestClientInternal implements SnowflakeStreamin
   // Snowflake role for the client to use
   private String role;
 
-  // Http client to send HTTP request to Snowflake
+  // Http client to send HTTP requests to Snowflake
   private final CloseableHttpClient httpClient;
 
   // Reference to the channel cache
@@ -132,6 +137,9 @@ public class SnowflakeStreamingIngestClientInternal implements SnowflakeStreamin
   // The request builder who handles building the HttpRequests we send
   private RequestBuilder requestBuilder;
 
+  // Background thread that uploads telemetry data periodically
+  private ScheduledExecutorService telemetryWorker;
+
   /**
    * Constructor
    *
@@ -141,7 +149,7 @@ public class SnowflakeStreamingIngestClientInternal implements SnowflakeStreamin
    * @param httpClient http client for sending request
    * @param isTestMode whether we're under test mode
    * @param requestBuilder http request builder
-   * @param parameterOverrides parameters we override incase we want to set different values
+   * @param parameterOverrides parameters we override in case we want to set different values
    */
   SnowflakeStreamingIngestClientInternal(
       String name,
@@ -167,18 +175,23 @@ public class SnowflakeStreamingIngestClientInternal implements SnowflakeStreamin
       try {
         KeyPair keyPair =
             Utils.createKeyPairFromPrivateKey((PrivateKey) prop.get(JDBC_PRIVATE_KEY));
-        this.requestBuilder = new RequestBuilder(accountURL, prop.get(USER).toString(), keyPair);
+        this.requestBuilder =
+            new RequestBuilder(
+                accountURL,
+                prop.get(USER).toString(),
+                keyPair,
+                this.httpClient,
+                String.format("%s_%s", this.name, System.currentTimeMillis()));
       } catch (NoSuchAlgorithmException | InvalidKeySpecException e) {
         throw new SFException(e, ErrorCode.KEYPAIR_CREATION_FAILURE);
       }
+
+      // Setup client telemetries if needed
+      this.setupMetricsForClient();
     }
 
     this.flushService = new FlushService(this, this.channelCache, this.isTestMode);
 
-    if (this.parameterProvider.hasEnabledSnowpipeStreamingMetrics()) {
-      this.registerMetricsForClient();
-    }
-
     logger.logInfo(
         "Client created, name={}, account={}. isTestMode={}, parameters={}",
         name,
@@ -250,9 +263,10 @@ public SnowflakeStreamingIngestChannelInternal openChannel(OpenChannelRequest re
     }
 
     logger.logDebug(
-        "Open channel request start, channel={}, table={}",
+        "Open channel request start, channel={}, table={}, client={}",
         request.getChannelName(),
-        request.getFullyQualifiedTableName());
+        request.getFullyQualifiedTableName(),
+        getName());
 
     try {
       Map<Object, Object> payload = new HashMap<>();
@@ -278,17 +292,19 @@ public SnowflakeStreamingIngestChannelInternal openChannel(OpenChannelRequest re
       // Check for Snowflake specific response code
       if (response.getStatusCode() != RESPONSE_SUCCESS) {
         logger.logDebug(
-            "Open channel request failed, channel={}, table={}, message={}",
+            "Open channel request failed, channel={}, table={}, client={}, message={}",
             request.getChannelName(),
             request.getFullyQualifiedTableName(),
+            getName(),
             response.getMessage());
         throw new SFException(ErrorCode.OPEN_CHANNEL_FAILURE, response.getMessage());
       }
 
-      logger.logDebug(
-          "Open channel request succeeded, channel={}, table={}",
+      logger.logInfo(
+          "Open channel request succeeded, channel={}, table={}, client={}",
           request.getChannelName(),
-          request.getFullyQualifiedTableName());
+          request.getFullyQualifiedTableName(),
+          getName());
 
       // Channel is now registered, add it to the in-memory channel pool
       SnowflakeStreamingIngestChannelInternal channel =
@@ -556,12 +572,17 @@ public void close() throws Exception {
     try {
       this.flush(true).get();
 
-      // unregister jmx metrics
+      // Report telemetry if needed
+      reportStreamingIngestTelemetryToSF();
+
+      // Unregister jmx metrics
       if (this.metrics != null) {
         Slf4jReporter.forRegistry(metrics).outputTo(logger.getLogger()).build().report();
         removeMetricsFromRegistry();
+      }
 
-        // LOG jvm memory and thread metrics at the end
+      // LOG jvm memory and thread metrics at the end
+      if (this.jvmMemoryAndThreadMetrics != null) {
         Slf4jReporter.forRegistry(jvmMemoryAndThreadMetrics)
             .outputTo(logger.getLogger())
             .build()
@@ -570,7 +591,13 @@ public void close() throws Exception {
     } catch (InterruptedException | ExecutionException e) {
       throw new SFException(e, ErrorCode.RESOURCE_CLEANUP_FAILURE, "client close");
     } finally {
+      if (this.telemetryWorker != null) {
+        this.telemetryWorker.shutdown();
+      }
       this.flushService.shutdown();
+      if (this.requestBuilder != null) {
+        this.requestBuilder.closeResources();
+      }
       Utils.closeAllocator(this.allocator);
     }
   }
@@ -730,49 +757,71 @@ ParameterProvider getParameterProvider() {
     return parameterProvider;
   }
 
-  /***
+  /**
    * Registers the performance metrics along with JVM memory and Threads.
    *
-   * Latency and throughput metrics are emitted to JMX, jvm memory and thread metrics are logged to Slf4JLogger
+   * <p>Latency and throughput metrics are emitted to JMX, jvm memory and thread metrics are logged
+   * to Slf4JLogger
    */
-  private void registerMetricsForClient() {
+  private void setupMetricsForClient() {
+    // Start the telemetry background worker if needed
+    if (ENABLE_TELEMETRY_TO_SF) {
+      this.telemetryWorker = Executors.newSingleThreadScheduledExecutor();
+      this.telemetryWorker.scheduleWithFixedDelay(
+          this::reportStreamingIngestTelemetryToSF,
+          STREAMING_INGEST_TELEMETRY_UPLOAD_INTERVAL_IN_SEC,
+          STREAMING_INGEST_TELEMETRY_UPLOAD_INTERVAL_IN_SEC,
+          TimeUnit.SECONDS);
+    }
+
+    // Register metrics if needed
     metrics = new MetricRegistry();
-    // Blob histograms
-    blobSizeHistogram = metrics.histogram(MetricRegistry.name("blob", "size", "histogram"));
-    blobRowCountHistogram =
-        metrics.histogram(MetricRegistry.name("blob", "row", "count", "histogram"));
-    cpuHistogram = metrics.histogram(MetricRegistry.name("cpu", "usage", "histogram"));
-
-    // latency metrics
-    flushLatency = metrics.timer(MetricRegistry.name("latency", "flush"));
-    buildLatency = metrics.timer(MetricRegistry.name("latency", "build"));
-    uploadLatency = metrics.timer(MetricRegistry.name("latency", "upload"));
-    registerLatency = metrics.timer(MetricRegistry.name("latency", "register"));
-
-    // throughput metrics
-    uploadThroughput = metrics.meter(MetricRegistry.name("throughput", "upload"));
-    inputThroughput = metrics.meter(MetricRegistry.name("throughput", "input"));
-    SharedMetricRegistries.add(SNOWPIPE_STREAMING_SHARED_METRICS_REGISTRY, metrics);
-
-    JmxReporter jmxReporter =
-        JmxReporter.forRegistry(this.metrics)
-            .inDomain(SNOWPIPE_STREAMING_JMX_METRIC_PREFIX)
-            .convertDurationsTo(TimeUnit.SECONDS)
-            .createsObjectNamesWith(
-                (ignoreMeterType, jmxDomain, metricName) ->
-                    getObjectName(this.getName(), jmxDomain, metricName))
-            .build();
-    jmxReporter.start();
-
-    // add JVM and thread metrics too
-    jvmMemoryAndThreadMetrics = new MetricRegistry();
-    jvmMemoryAndThreadMetrics.register(
-        MetricRegistry.name("jvm", "memory"), new MemoryUsageGaugeSet());
-    jvmMemoryAndThreadMetrics.register(
-        MetricRegistry.name("jvm", "threads"), new ThreadStatesGaugeSet());
-
-    SharedMetricRegistries.add(
-        SNOWPIPE_STREAMING_JVM_MEMORY_AND_THREAD_METRICS_REGISTRY, jvmMemoryAndThreadMetrics);
+
+    if (ENABLE_TELEMETRY_TO_SF || this.parameterProvider.hasEnabledSnowpipeStreamingMetrics()) {
+      // CPU usage metric
+      cpuHistogram = metrics.histogram(MetricRegistry.name("cpu", "usage", "histogram"));
+
+      // Latency metrics
+      flushLatency = metrics.timer(MetricRegistry.name("latency", "flush"));
+      buildLatency = metrics.timer(MetricRegistry.name("latency", "build"));
+      uploadLatency = metrics.timer(MetricRegistry.name("latency", "upload"));
+      registerLatency = metrics.timer(MetricRegistry.name("latency", "register"));
+
+      // Throughput metrics
+      uploadThroughput = metrics.meter(MetricRegistry.name("throughput", "upload"));
+      inputThroughput = metrics.meter(MetricRegistry.name("throughput", "input"));
+
+      // Blob histogram metrics
+      blobSizeHistogram = metrics.histogram(MetricRegistry.name("blob", "size", "histogram"));
+      blobRowCountHistogram =
+          metrics.histogram(MetricRegistry.name("blob", "row", "count", "histogram"));
+    }
+
+    if (this.parameterProvider.hasEnabledSnowpipeStreamingMetrics()) {
+      JmxReporter jmxReporter =
+          JmxReporter.forRegistry(this.metrics)
+              .inDomain(SNOWPIPE_STREAMING_JMX_METRIC_PREFIX)
+              .convertDurationsTo(TimeUnit.SECONDS)
+              .createsObjectNamesWith(
+                  (ignoreMeterType, jmxDomain, metricName) ->
+                      getObjectName(this.getName(), jmxDomain, metricName))
+              .build();
+      jmxReporter.start();
+
+      // Add JVM and thread metrics too
+      jvmMemoryAndThreadMetrics = new MetricRegistry();
+      jvmMemoryAndThreadMetrics.register(
+          MetricRegistry.name("jvm", "memory"), new MemoryUsageGaugeSet());
+      jvmMemoryAndThreadMetrics.register(
+          MetricRegistry.name("jvm", "threads"), new ThreadStatesGaugeSet());
+
+      SharedMetricRegistries.add(
+          SNOWPIPE_STREAMING_JVM_MEMORY_AND_THREAD_METRICS_REGISTRY, jvmMemoryAndThreadMetrics);
+    }
+
+    if (metrics.getMetrics().size() != 0) {
+      SharedMetricRegistries.add(SNOWPIPE_STREAMING_SHARED_METRICS_REGISTRY, metrics);
+    }
   }
 
   /**
@@ -784,7 +833,7 @@ private void registerMetricsForClient() {
    * @param metricName metric name used while registering the metric.
    * @return Object Name constructed from above three args
    */
-  static ObjectName getObjectName(String clientName, String jmxDomain, String metricName) {
+  private static ObjectName getObjectName(String clientName, String jmxDomain, String metricName) {
     try {
       String sb = jmxDomain + ":clientName=" + clientName + ",name=" + metricName;
 
@@ -803,4 +852,24 @@ private void removeMetricsFromRegistry() {
       SharedMetricRegistries.remove(SNOWPIPE_STREAMING_SHARED_METRICS_REGISTRY);
     }
   }
+
+  /**
+   * Get the Telemetry Service for a given client
+   *
+   * @return TelemetryService used by the client
+   */
+  TelemetryService getTelemetryService() {
+    return this.requestBuilder == null ? null : requestBuilder.getTelemetryService();
+  }
+
+  /** Report streaming ingest related telemetries to Snowflake */
+  private void reportStreamingIngestTelemetryToSF() {
+    TelemetryService telemetryService = getTelemetryService();
+    if (telemetryService != null) {
+      telemetryService.reportLatencyInSec(
+          this.buildLatency, this.uploadLatency, this.registerLatency, this.flushLatency);
+      telemetryService.reportThroughputBytesPerSecond(this.inputThroughput, this.uploadThroughput);
+      telemetryService.reportCpuMemoryUsage(this.cpuHistogram);
+    }
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index 8a23845b5..f9d2eb4a3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -29,6 +29,7 @@
 import net.snowflake.client.jdbc.SnowflakeSQLException;
 import net.snowflake.client.jdbc.cloud.storage.StageInfo;
 import net.snowflake.client.jdbc.internal.apache.commons.io.FileUtils;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
@@ -38,7 +39,6 @@
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.Utils;
 import org.apache.arrow.util.VisibleForTesting;
-import org.apache.http.impl.client.CloseableHttpClient;
 
 /** Handles uploading files to the Snowflake Streaming Ingest Stage */
 class StreamingIngestStage {
@@ -56,7 +56,7 @@ static class SnowflakeFileTransferMetadataWithAge {
     private final boolean isLocalFS;
     private final String localLocation;
 
-    /* Do do not always know the age of the metadata, so we use the empty
+    /* Do not always know the age of the metadata, so we use the empty
     state to record unknown age.
      */
     Optional<Long> timestamp;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
index d0c0f3a96..db6ac4e61 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
@@ -8,14 +8,14 @@
 import java.io.IOException;
 import java.util.Map;
 import java.util.function.Function;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.ingest.connection.IngestResponseException;
 import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.connection.ServiceResponseHandler;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.SFException;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.impl.client.CloseableHttpClient;
 
 public class StreamingIngestUtils {
 
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index bd1691efe..8aa09868b 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -50,6 +50,7 @@ public class Constants {
   public static final String ENCRYPTION_ALGORITHM = "AES/CTR/NoPadding";
   public static final int ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES = 16;
   public static final int MAX_STREAMING_INGEST_API_CHANNEL_RETRY = 3;
+  public static final int STREAMING_INGEST_TELEMETRY_UPLOAD_INTERVAL_IN_SEC = 10;
 
   // Channel level constants
   public static final String CHANNEL_STATUS_ENDPOINT = "/v1/streaming/channels/status/";
@@ -77,7 +78,7 @@ public enum ArrowBatchWriteMode {
     FILE,
   }
 
-  /** Thw write mode to generate Arrow BDEC file. */
+  /** The write mode to generate Arrow BDEC file. */
   public enum BdecVerion {
     /** Uses Arrow to generate BDEC chunks with {@link ArrowBatchWriteMode#STREAM}. */
     ONE(1),
@@ -119,6 +120,7 @@ public static BdecVerion fromInt(int val) {
   public static final boolean DISABLE_BACKGROUND_FLUSH = false;
   public static final boolean COMPRESS_BLOB_TWICE = false;
   public static final boolean BLOB_NO_HEADER = true;
+  public static final boolean ENABLE_TELEMETRY_TO_SF = true;
 
   // Metrics
   public static final String SNOWPIPE_STREAMING_JMX_METRIC_PREFIX = "snowflake.ingest.sdk";
diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index ad1139974..f96a32489 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -13,29 +13,29 @@
 import java.util.concurrent.locks.ReentrantLock;
 import javax.net.ssl.SSLContext;
 import net.snowflake.client.core.SFSessionProperty;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpRequest;
-import org.apache.http.HttpResponse;
-import org.apache.http.NoHttpResponseException;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.Credentials;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.client.HttpRequestRetryHandler;
-import org.apache.http.client.ServiceUnavailableRetryStrategy;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.conn.routing.HttpRoute;
-import org.apache.http.conn.ssl.DefaultHostnameVerifier;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.impl.conn.DefaultProxyRoutePlanner;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.http.pool.PoolStats;
-import org.apache.http.protocol.HttpContext;
-import org.apache.http.ssl.SSLContexts;
+import net.snowflake.client.jdbc.internal.apache.http.HttpHost;
+import net.snowflake.client.jdbc.internal.apache.http.HttpRequest;
+import net.snowflake.client.jdbc.internal.apache.http.HttpResponse;
+import net.snowflake.client.jdbc.internal.apache.http.NoHttpResponseException;
+import net.snowflake.client.jdbc.internal.apache.http.auth.AuthScope;
+import net.snowflake.client.jdbc.internal.apache.http.auth.Credentials;
+import net.snowflake.client.jdbc.internal.apache.http.auth.UsernamePasswordCredentials;
+import net.snowflake.client.jdbc.internal.apache.http.client.CredentialsProvider;
+import net.snowflake.client.jdbc.internal.apache.http.client.HttpRequestRetryHandler;
+import net.snowflake.client.jdbc.internal.apache.http.client.ServiceUnavailableRetryStrategy;
+import net.snowflake.client.jdbc.internal.apache.http.client.config.RequestConfig;
+import net.snowflake.client.jdbc.internal.apache.http.client.protocol.HttpClientContext;
+import net.snowflake.client.jdbc.internal.apache.http.conn.routing.HttpRoute;
+import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.DefaultHostnameVerifier;
+import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.BasicCredentialsProvider;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.HttpClientBuilder;
+import net.snowflake.client.jdbc.internal.apache.http.impl.conn.DefaultProxyRoutePlanner;
+import net.snowflake.client.jdbc.internal.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import net.snowflake.client.jdbc.internal.apache.http.pool.PoolStats;
+import net.snowflake.client.jdbc.internal.apache.http.protocol.HttpContext;
+import net.snowflake.client.jdbc.internal.apache.http.ssl.SSLContexts;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -60,7 +60,7 @@ public class HttpUtil {
    * This lock is to synchronize on idleConnectionMonitorThread to avoid setting starting a thread
    * which was already started. (To avoid {@link IllegalThreadStateException})
    */
-  private static ReentrantLock idleConnectionMonitorThreadLock = new ReentrantLock(true);
+  private static final ReentrantLock idleConnectionMonitorThreadLock = new ReentrantLock(true);
 
   private static final int DEFAULT_CONNECTION_TIMEOUT_MINUTES = 1;
   private static final int DEFAULT_HTTP_CLIENT_SOCKET_TIMEOUT_MINUTES = 5;
@@ -125,10 +125,8 @@ private static void initHttpClient() {
     connectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_CONNECTIONS_PER_ROUTE);
     connectionManager.setMaxTotal(DEFAULT_MAX_CONNECTIONS);
 
-    /**
-     * Use a anonymous class to implement the interface ServiceUnavailableRetryStrategy() The max
-     * retry time is 3. The interval time is backoff.
-     */
+    // Use an anonymous class to implement the interface ServiceUnavailableRetryStrategy() The max
+    // retry time is 3. The interval time is backoff.
     HttpClientBuilder clientBuilder =
         HttpClientBuilder.create()
             .setConnectionManager(connectionManager)
diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
index 207db28a7..77a00bdae 100644
--- a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
@@ -20,6 +20,9 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import net.snowflake.client.jdbc.internal.apache.http.Header;
+import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost;
 import net.snowflake.ingest.connection.ClientStatusResponse;
 import net.snowflake.ingest.connection.ConfigureClientResponse;
 import net.snowflake.ingest.connection.HistoryResponse;
@@ -27,9 +30,6 @@
 import net.snowflake.ingest.connection.IngestResponseException;
 import net.snowflake.ingest.connection.InsertFilesClientInfo;
 import net.snowflake.ingest.utils.StagedFileWrapper;
-import org.apache.http.Header;
-import org.apache.http.HttpHeaders;
-import org.apache.http.client.methods.HttpPost;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
diff --git a/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java b/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java
deleted file mode 100644
index a7570f4c7..000000000
--- a/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java
+++ /dev/null
@@ -1,186 +0,0 @@
-package net.snowflake.ingest.connection;
-
-import static org.junit.Assert.assertTrue;
-
-import java.security.KeyFactory;
-import java.security.KeyPair;
-import java.security.NoSuchAlgorithmException;
-import java.security.PrivateKey;
-import java.security.PublicKey;
-import java.security.spec.InvalidKeySpecException;
-import java.security.spec.PKCS8EncodedKeySpec;
-import java.security.spec.X509EncodedKeySpec;
-import java.util.Arrays;
-import java.util.Base64;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class SecurityManagerTest {
-  private String expectedPublicKeyFp = "SHA256:yVUGJrOo4BN1Cza+m2zNzvQbk/4rICTydzSNvuiyy9Q=";
-
-  /** BASE64 ENCODED PUBLIC KEY */
-  private String storedPublicKey =
-      "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtRPru42llC40VdmWnc8r\n"
-          + "7TI/AFemZw4Lh1HRnPIFRxwhOE/yxHHxFGuPLUouyHWM9rVT9N9eo6PTOB8TCnGw\n"
-          + "fwTW2jloSbjtycDdM3+UrBUpX7x/Ufhcwoeu0O3NR5pAhGJRVKCvSpmrD3k2l2vZ\n"
-          + "sRL0230IPGxeDB8m2Wia8QCKKou7AkSsmQ3/9kcKowLGf2axPHty2QSXx4NKvwe0\n"
-          + "B1NnLcQTBc6Z83Lym3gKn8YSINk1ZoO9G5oQKr64wnuQIOlXjcXD8BAEYKbv7VkG\n"
-          + "1vsikqixFpPfWrGUlzhoWWTcn4awzRFaX81ZzhAtfA/laqVvrqN/+O6Cc1k614kV\n"
-          + "GQIDAQAB\n";
-
-  /** BASE64 ENCODED PRIVATE KEY */
-  private String storedPrivateKey =
-      "MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC1E+u7jaWULjRV\n"
-          + "2ZadzyvtMj8AV6ZnDguHUdGc8gVHHCE4T/LEcfEUa48tSi7IdYz2tVP0316jo9M4\n"
-          + "HxMKcbB/BNbaOWhJuO3JwN0zf5SsFSlfvH9R+FzCh67Q7c1HmkCEYlFUoK9KmasP\n"
-          + "eTaXa9mxEvTbfQg8bF4MHybZaJrxAIoqi7sCRKyZDf/2RwqjAsZ/ZrE8e3LZBJfH\n"
-          + "g0q/B7QHU2ctxBMFzpnzcvKbeAqfxhIg2TVmg70bmhAqvrjCe5Ag6VeNxcPwEARg\n"
-          + "pu/tWQbW+yKSqLEWk99asZSXOGhZZNyfhrDNEVpfzVnOEC18D+VqpW+uo3/47oJz\n"
-          + "WTrXiRUZAgMBAAECggEAd/XnkMwJKr7law6IaqmqJyzHchmfIty6JH6+yCPJ/U8U\n"
-          + "bvMAGMaHeQi8xLtFfQXrSjHcmfg1AWHx91cWzS9+RtfU4qNvhI+f8K31nT1jKBGo\n"
-          + "5ETDcHGCOlmbJcy01z/IiCt+R/tfxaNCITEjSnNnt5igYJxXjXgZYhxtJ1DWfNvJ\n"
-          + "0PEPRF+Wuo1y77EW4ACzAKdZriLuJ+ynakYIZc8TC+w2ZiSDHx0ZEQ2YmV7m7A11\n"
-          + "db1bAw4X7Z7ECvx9VaZKmvj3v9xa8BIcJtYw8YJSHL0f3g/xz+qAqDVw5X7zofdO\n"
-          + "FEwx96/o14lWnohVjruaGEQvMsNKNz/ONQ1cUaTsFQKBgQDlik5hjo7AEx8mlu5q\n"
-          + "vXt9iAqSSSI56bunJofbfRLihI9Z3e10v5pvdj9CwB/gJYtOJ4rYLo4KGY7AMq7r\n"
-          + "q4ObVGF3lIIdBQldy0B+w2jHaemNlg5shQKqC4dA26MlOKO0iIvS1HIazkRGk1sg\n"
-          + "p8NVaG7jMvGyzD75UsdPRFoBxwKBgQDJ838J+R2OjAdbcDKblfpc/O+IU5chxNsw\n"
-          + "ojlA4NLbROr9RBNj04r565g1kU0vbj14Cj9Ocifb9yZNQJPIPfyVVg8LIrt9YciC\n"
-          + "GnvUSdX/558+O7Y/HwqXlzGt3TRnHpdH34qO/CSUEl5kDP/TmW3sw6VNYZa1QwhW\n"
-          + "VldKgWYyHwKBgDzUomEIPpx4dNDtPtHa1Vc3LlYGO6PNZYWumGJ6iv6s0rCmN7+w\n"
-          + "52SSmcE+2TO1v20+3XTdIZdbnpEg3WpnUcFgY1QlbzXxl8Hbk4QElUgDsXlsQvZP\n"
-          + "aZ1W4Mk3a8z5bajyZtvAoVypPT7W3leRHhsMSha78YHIzweUAG3pV1ERAoGATEg4\n"
-          + "nVjG7FhCUyyvQQvGtScph3IjrTLBpL4yKCqEGyUOKjpzpIp8fWibZuiKojbe6x/b\n"
-          + "x9Lg8XqKsjWJXOLlLLeEGS22ambsKRC943M8bVxdT1GYxoEALECFGGps5+KrPA/Z\n"
-          + "M6dUXcYOd3Zdj9zto7hHEVKibbdzR8F3WYJFSvsCgYBj1TqNONCQRJvo298bJtm9\n"
-          + "RXzYQvjHXnNNnbEL+B+B/r2jBOM/t8WBLJVNghQDhJY+DMMdptxkvNge3XNenDtY\n"
-          + "25UsnB7XefRE0tDe6yLWKbONT33+WGZjCBCXpQ90Avvi9npFetwG9Q8GSP7VdbTc\n"
-          + "tLwfZra8DDXs5Dz9Gion+A==\n";
-
-  private String expectedPublicKeyFp2 = "SHA256:MDcEdlQsgzIs7UBLHV6CB9GJLqqW/AqGsMcAPrWVxuA=";
-
-  private String storedPublicKey2 =
-      "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwwtuB1ZFqe9jF8TjvwoH\n"
-          + "oGO2KRxSu8GRDuI93/g1dJKve/SsmNB+SPQ402tEmnejg6QMlyFOvh+bbEqYooXN\n"
-          + "6lCaFSk7DAx7aNqr1yU/Jpwzbal0H74PhOAw4u1iyBCf08r8aQHLYkOUF2DcggUI\n"
-          + "WCKrBnpEC6vK8aZRWGwpgXB46CkousWXrmKBqbEBJusj2/Fgrk2CZ/OGY/vlzh6A\n"
-          + "7TpucviZUF3bsmsEs//63XpTSQsL785uixJbnQye8HDN4iyjvK09dHruIfSVPZ2N\n"
-          + "7xPdw7Nvyf+gRBmu2HWCPFpOc7a0XPNarlQPXPLbGz47dIZNEW+8p2jdw1D2PZ3h\n"
-          + "vwIDAQAB\n";
-
-  private String storedPrivateKey2 =
-      "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDDC24HVkWp72MX\n"
-          + "xOO/CgegY7YpHFK7wZEO4j3f+DV0kq979KyY0H5I9DjTa0Sad6ODpAyXIU6+H5ts\n"
-          + "Spiihc3qUJoVKTsMDHto2qvXJT8mnDNtqXQfvg+E4DDi7WLIEJ/TyvxpActiQ5QX\n"
-          + "YNyCBQhYIqsGekQLq8rxplFYbCmBcHjoKSi6xZeuYoGpsQEm6yPb8WCuTYJn84Zj\n"
-          + "++XOHoDtOm5y+JlQXduyawSz//rdelNJCwvvzm6LEludDJ7wcM3iLKO8rT10eu4h\n"
-          + "9JU9nY3vE93Ds2/J/6BEGa7YdYI8Wk5ztrRc81quVA9c8tsbPjt0hk0Rb7ynaN3D\n"
-          + "UPY9neG/AgMBAAECggEBAL9JLGXBtJzPDC45iOrJWWVxpSt4faNqWWtxcyF++l4T\n"
-          + "pks5UTSl9dRywHCImUWs5A6tCzQFFIbd1L5GAqAR/js5RYRPZXuRmk7hdvqPqvmg\n"
-          + "48c/E4Y2Dl5QyWElU2XG+Bjs0NPjUKZUhJ7Q/jH94YsepQC7VJTlrSmF5e2EVsh7\n"
-          + "3wwAd+fRNSvWmMoVsjw1gX8Cen8rCEZZyPSwaK1pYBQga0abEZ6BdQQ0O3yTZQga\n"
-          + "Q3z+vC+hm3cqpAmhWupYxqdA/EBNz/v0GVe8BGbEYAnm/4LPpQilt1Z1ngNqdUoT\n"
-          + "zF4PXnYQ/fFOR90nbjWoJLJYCDd+QW4XXwxwOIs/BsECgYEA+USiF2F+lG7y6dhk\n"
-          + "fLAi8UROAjYv+/wWSBnOV0JCD0A6Ik5bOXvXs72/NCHdBGq93mbYhzo7ar9Fm0QE\n"
-          + "mtwfnfhVNRk+SSq/+hzCOmZrdZpu2nMzBDJtR2TartOwfijE/NNZGA1V7QOmR02u\n"
-          + "WINyaqvDNNPVIt+D6qreXcZDslECgYEAyE/pWfLjJfWBXtOruXxBcJKq5Arx0YSv\n"
-          + "i7VaiOg9zBtLg4imKvRJqjs6Y7rGap3qA+HguEl4TQHvhIfhlAxWJ2XVB0ijTAiF\n"
-          + "aMTmBmxCJq0V9raScJPbSHAN54T2f0x+yh5/q7IFFKF2Hn3qrmarSAw02G+KTH6M\n"
-          + "lFFqJ+Dcvw8CgYEA7aMX4LBqq3nGjVdmHVUSSu7ya7tbLaDbYStxAtFBBycVBQWs\n"
-          + "hHXjYxD/SuUJvx9AGdn0jZ7fbFojMu26ciRu4/wOx4tkTP67fOeT53ci9UAgdJQk\n"
-          + "y9iDQ/ALZ2abOPsHKX0X0A1OoKG9EPcmwm22U6midSeKZy+tpLf3PHE6srECgYAa\n"
-          + "yj0+T3K7t+r2gL69zvV9ldAPMbuHtwQ3XijemJjzPE9MJzF6GzPi9Yronak9xyLu\n"
-          + "I/6HByR0wCaFhhrQTxoSqNbl43wbhiQ5j+PnxgDO5WVDmsVZEx1HwdzKMwk4m0V1\n"
-          + "yMBweR2e1b1TdKm3a3nK5/8FV12av24TxBO7g6JiVwKBgDnaedWvgJt+tJrkFn9h\n"
-          + "ZE2VVGC4FjFDpCMxIebvBz5Kbs+lknpdcda8+DMwXFDf8OR3lt/3KzJqleUhrTTz\n"
-          + "AQz56Xdi9VEnVs3rsgvX9VnaWcRpa4GT5EIj+I2M9t+D8XCfMMs1S56Pnn5oGkqv\n"
-          + "FBzmMRnskqK6d75B8EG5BGi0\n";
-
-  @Test
-  public void validatePublicKeyFp() throws NoSuchAlgorithmException, InvalidKeySpecException {
-
-    PublicKey pubKey = loadPublicKey(storedPublicKey);
-    PrivateKey priKey = loadPrivateKey(storedPrivateKey);
-
-    KeyPair keypair = new KeyPair(pubKey, priKey);
-
-    String accountName = "accountName";
-    String userName = "userName";
-    SecurityManager securityManager = new SecurityManager(accountName, userName, keypair);
-    String publicKeyFp = securityManager.getPublicKeyFingerPrint();
-    assertTrue(publicKeyFp.equals(expectedPublicKeyFp));
-
-    PublicKey pubKey2 = loadPublicKey(storedPublicKey2);
-    PrivateKey priKey2 = loadPrivateKey(storedPrivateKey2);
-
-    KeyPair keypair2 = new KeyPair(pubKey2, priKey2);
-
-    SecurityManager securityManager2 = new SecurityManager(accountName, userName, keypair2);
-    String publicKeyFp2 = securityManager2.getPublicKeyFingerPrint();
-    assertTrue(publicKeyFp2.equals(expectedPublicKeyFp2));
-  }
-
-  @Test
-  public void testParseAccount() throws NoSuchAlgorithmException, InvalidKeySpecException {
-    PublicKey pubKey = loadPublicKey(storedPublicKey);
-    PrivateKey priKey = loadPrivateKey(storedPrivateKey);
-
-    KeyPair keypair = new KeyPair(pubKey, priKey);
-
-    String accountName = "accountName";
-    String userName = "userName";
-    SecurityManager securityManager = new SecurityManager(accountName, userName, keypair);
-    Assert.assertEquals(accountName.toUpperCase(), securityManager.getAccount());
-  }
-
-  @Test
-  public void testParseAccount_dotInAccountName()
-      throws NoSuchAlgorithmException, InvalidKeySpecException {
-    PublicKey pubKey = loadPublicKey(storedPublicKey);
-    PrivateKey priKey = loadPrivateKey(storedPrivateKey);
-
-    KeyPair keypair = new KeyPair(pubKey, priKey);
-
-    String accountName = "accountName.extra";
-    String userName = "userName";
-    String trimmedAccountName = "accountName";
-    SecurityManager securityManager = new SecurityManager(accountName, userName, keypair);
-    Assert.assertEquals(trimmedAccountName.toUpperCase(), securityManager.getAccount());
-  }
-
-  /**
-   * Converts encodedBase64 publicKey back to the RSA scheme PublicKey object.
-   *
-   * <p>
-   *
-   * @param base64PublicKey
-   * @return
-   * @throws NoSuchAlgorithmException
-   * @throws InvalidKeySpecException
-   */
-  private PublicKey loadPublicKey(String base64PublicKey)
-      throws NoSuchAlgorithmException, InvalidKeySpecException {
-    byte[] data = Base64.getMimeDecoder().decode(base64PublicKey);
-    X509EncodedKeySpec spec = new X509EncodedKeySpec(data);
-    KeyFactory factory = KeyFactory.getInstance("RSA");
-    return factory.generatePublic(spec);
-  }
-
-  /**
-   * Converts privateKey in encodedBase64 in String to RSA scheme PrivateKey.
-   *
-   * @param base64PrivateKey
-   * @return
-   * @throws NoSuchAlgorithmException
-   * @throws InvalidKeySpecException
-   */
-  private PrivateKey loadPrivateKey(String base64PrivateKey)
-      throws NoSuchAlgorithmException, InvalidKeySpecException {
-    byte[] clear = Base64.getMimeDecoder().decode(base64PrivateKey);
-    PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(clear);
-    KeyFactory factory = KeyFactory.getInstance("RSA");
-    PrivateKey privateKey = factory.generatePrivate(keySpec);
-    Arrays.fill(clear, (byte) 0);
-    return privateKey;
-  }
-}
diff --git a/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java b/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java
new file mode 100644
index 000000000..962aa1a81
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java
@@ -0,0 +1,76 @@
+package net.snowflake.ingest.connection;
+
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class TelemetryServiceTest {
+  @Test
+  public void testReportLatencyInSec() {
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+
+    TelemetryService telemetryService =
+        Mockito.spy(
+            new TelemetryService(httpClient, "testReportLatencyInSec", "snowflake.dev.local:8082"));
+    Mockito.doNothing().when(telemetryService).send(Mockito.any(), Mockito.any());
+    MetricRegistry metrics = new MetricRegistry();
+    Timer flushLatency = metrics.timer(MetricRegistry.name("latency", "flush"));
+    Timer buildLatency = metrics.timer(MetricRegistry.name("latency", "build"));
+    Timer uploadLatency = metrics.timer(MetricRegistry.name("latency", "upload"));
+    Timer registerLatency = metrics.timer(MetricRegistry.name("latency", "register"));
+
+    // Make sure there is no exception thrown
+    telemetryService.reportLatencyInSec(buildLatency, uploadLatency, registerLatency, flushLatency);
+  }
+
+  @Test
+  public void testReportClientFailure() {
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+
+    TelemetryService telemetryService =
+        Mockito.spy(
+            new TelemetryService(
+                httpClient, "testReportClientFailure", "snowflake.dev.local:8082"));
+    Mockito.doNothing().when(telemetryService).send(Mockito.any(), Mockito.any());
+
+    // Make sure there is no exception thrown
+    telemetryService.reportClientFailure("testReportClientFailure", "exception");
+  }
+
+  @Test
+  public void testReportThroughputBytesPerSecond() {
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+
+    TelemetryService telemetryService =
+        Mockito.spy(
+            new TelemetryService(
+                httpClient, "testReportThroughputBytesPerSecond", "snowflake.dev.local:8082"));
+    Mockito.doNothing().when(telemetryService).send(Mockito.any(), Mockito.any());
+    MetricRegistry metrics = new MetricRegistry();
+    Meter uploadThroughput = metrics.meter(MetricRegistry.name("throughput", "upload"));
+    Meter inputThroughput = metrics.meter(MetricRegistry.name("throughput", "input"));
+
+    // Make sure there is no exception thrown
+    telemetryService.reportThroughputBytesPerSecond(uploadThroughput, inputThroughput);
+  }
+
+  @Test
+  public void testReportCpuMemoryUsage() {
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+
+    TelemetryService telemetryService =
+        Mockito.spy(
+            new TelemetryService(
+                httpClient, "testReportCpuMemoryUsage", "snowflake.dev.local:8082"));
+    Mockito.doNothing().when(telemetryService).send(Mockito.any(), Mockito.any());
+    MetricRegistry metrics = new MetricRegistry();
+    Histogram cpuHistogram = metrics.histogram(MetricRegistry.name("cpu", "usage", "histogram"));
+
+    // Make sure there is no exception thrown
+    telemetryService.reportCpuMemoryUsage(cpuHistogram);
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/connection/TestKeyRenewal.java b/src/test/java/net/snowflake/ingest/connection/TestKeyRenewal.java
index 351de1e21..9bc653c39 100644
--- a/src/test/java/net/snowflake/ingest/connection/TestKeyRenewal.java
+++ b/src/test/java/net/snowflake/ingest/connection/TestKeyRenewal.java
@@ -30,7 +30,8 @@ public void doesRegenerateToken()
     KeyPair keyPair = keyGen.generateKeyPair();
 
     // create the security manager;
-    SecurityManager manager = new SecurityManager("account", "user", keyPair, 3, TimeUnit.SECONDS);
+    SecurityManager manager =
+        new SecurityManager("account", "user", keyPair, 3, TimeUnit.SECONDS, null);
 
     // grab a token
     String token = manager.getToken();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index a68f5db7b..0857dcc76 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -19,6 +19,12 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils;
+import net.snowflake.client.jdbc.internal.apache.http.HttpEntity;
+import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders;
+import net.snowflake.client.jdbc.internal.apache.http.StatusLine;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.ingest.TestUtils;
 import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
@@ -30,12 +36,6 @@
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.SnowflakeURL;
 import net.snowflake.ingest.utils.Utils;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHeaders;
-import org.apache.http.StatusLine;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.impl.client.CloseableHttpClient;
 import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -247,7 +247,8 @@ public void testOpenChannelPostRequest() throws Exception {
     SnowflakeURL url = new SnowflakeURL(urlStr);
 
     KeyPair keyPair = Utils.createKeyPairFromPrivateKey((PrivateKey) prop.get(JDBC_PRIVATE_KEY));
-    RequestBuilder requestBuilder = new RequestBuilder(url, prop.get(USER).toString(), keyPair);
+    RequestBuilder requestBuilder =
+        new RequestBuilder(url, prop.get(USER).toString(), keyPair, null, null);
 
     Map<Object, Object> payload = new HashMap<>();
     payload.put("channel", "CHANNEL");
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index c9d754632..f651cba10 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -31,6 +31,12 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils;
+import net.snowflake.client.jdbc.internal.apache.http.HttpEntity;
+import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders;
+import net.snowflake.client.jdbc.internal.apache.http.StatusLine;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.asn1.nist.NISTObjectIdentifiers;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
@@ -50,12 +56,6 @@
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.SnowflakeURL;
 import net.snowflake.ingest.utils.Utils;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHeaders;
-import org.apache.http.StatusLine;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.impl.client.CloseableHttpClient;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -426,7 +426,8 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
     SnowflakeURL url = new SnowflakeURL(urlStr);
 
     KeyPair keyPair = Utils.createKeyPairFromPrivateKey((PrivateKey) prop.get(JDBC_PRIVATE_KEY));
-    RequestBuilder requestBuilder = new RequestBuilder(url, prop.get(USER).toString(), keyPair);
+    RequestBuilder requestBuilder =
+        new RequestBuilder(url, prop.get(USER).toString(), keyPair, null, null);
 
     SnowflakeStreamingIngestChannelInternal channel =
         new SnowflakeStreamingIngestChannelInternal(
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
index e1eda268f..500791673 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
@@ -31,6 +31,10 @@
 import net.snowflake.client.jdbc.SnowflakeSQLException;
 import net.snowflake.client.jdbc.cloud.storage.StageInfo;
 import net.snowflake.client.jdbc.internal.amazonaws.util.IOUtils;
+import net.snowflake.client.jdbc.internal.apache.http.StatusLine;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse;
+import net.snowflake.client.jdbc.internal.apache.http.entity.BasicHttpEntity;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
 import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -38,10 +42,6 @@
 import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ParameterProvider;
-import org.apache.http.StatusLine;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.entity.BasicHttpEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
index 6f1653dc8..c0807bb81 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
@@ -10,12 +10,12 @@
 import java.io.InputStream;
 import java.util.ArrayList;
 import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils;
+import net.snowflake.client.jdbc.internal.apache.http.HttpEntity;
+import net.snowflake.client.jdbc.internal.apache.http.StatusLine;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.ingest.TestUtils;
 import net.snowflake.ingest.connection.RequestBuilder;
-import org.apache.http.HttpEntity;
-import org.apache.http.StatusLine;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.impl.client.CloseableHttpClient;
 import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.Mockito;

From 44c9d1e2621e056cd74dd7513a08e0bef36e98d1 Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Tue, 26 Jul 2022 11:48:53 +0000
Subject: [PATCH 048/356] Disable sending blob_version to server side until
 supported in prod

---
 .../ingest/streaming/internal/BlobMetadata.java          | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index 7ae2eca76..7199c664d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -49,8 +49,9 @@ List<ChunkMetadata> getChunks() {
     return this.chunks;
   }
 
-  @JsonProperty("bdec_version")
-  byte getVersionByte() {
-    return bdecVersion.toByte();
-  }
+  // TODO: send the bdec_version once server side supports this in production
+  //  @JsonProperty("bdec_version")
+  //  byte getVersionByte() {
+  //    return bdecVersion.toByte();
+  //  }
 }

From 1034d46a0ce203dd5efbf583ee4a31401ec51147 Mon Sep 17 00:00:00 2001
From: Konstantinos Kloudas <konstantinos.kloudas@snowflake.com>
Date: Fri, 29 Jul 2022 02:27:06 +0200
Subject: [PATCH 049/356] SNOW-593873 SNOW-636695 Round timestamp to seconds
 when creating blob name (#203)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

During investigation of the parent issue, we saw that bdec file shortnames look like l5kkhkwc_HFDUCTv22sdxqQR6lW9nOq9dg5e2aiSnSgo5exuHUqACC_22_144.bdec while fdn shortnames look like reyy2e_316685135989106_1_001 .

In the scenario where the two above files end up in the same EpFile , that EpFile will have reyy2e_316685135989106_1_001 as its max fdn shortname and l5kkhkwc_HFDUCTv22sdxqQR6lW9nOq9dg5e2aiSnSgo5exuHUqACC_22_144.bdec as its min, as they are compared lexicographically.

When computing the EpFileBucketIndex (e.g. during any pruning phase of any query or DML), we bucketize EpFiles based on their min/max fdnshortnames. To do this we extract the base36 encoded timestamp in the beginning of each shortname and we compute the actual timestamp in ms. In the case above, we have reyy2e = 1657731830 while l5kkhkwc = 1657775093772 which is greater. The reason for this is that the timestamp used in the FDN shortname creation in seconds and NOT in milliseconds, as is the case for BDEC’s. So the min shortname seems to have a bigger timestamp than the max shortname.

This leads to such EpFiles always been assigned to the bucket in the EpFileBucketIndex with index 0 and if, as in our bug, a deleted BDEC file is in a compacted EpFile that also has FDN files (so it ends up in index 0 of the index) but the BDEC file itself is assigned to another bucket, then the EpFile will not be scanned.
---
 .../net/snowflake/ingest/streaming/internal/FlushService.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 4f1974e8a..7d0860bdb 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -663,7 +663,7 @@ String getFilePath(Calendar calendar, String clientPrefix) {
     int day = calendar.get(Calendar.DAY_OF_MONTH);
     int hour = calendar.get(Calendar.HOUR_OF_DAY);
     int minute = calendar.get(Calendar.MINUTE);
-    long time = calendar.getTimeInMillis();
+    long time = TimeUnit.MILLISECONDS.toSeconds(calendar.getTimeInMillis());
     long threadId = Thread.currentThread().getId();
     String fileName =
         Long.toString(time, 36)

From 6b0da390e1067b5f4bf4e6532e24a5e31060d5f0 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 28 Jul 2022 17:30:33 -0700
Subject: [PATCH 050/356] new release

---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 52a35f01b..b4672607c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
     <!-- Arifact name and version information -->
     <groupId>net.snowflake</groupId>
     <artifactId>snowflake-ingest-sdk</artifactId>
-    <version>1.0.2-beta.2</version>
+    <version>1.0.2-beta.3</version>
     <packaging>jar</packaging>
     <name>Snowflake Ingest SDK</name>
     <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 0ff50f6d5..f4edc66d7 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -127,7 +127,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.0.2-beta.2";
+  public static final String DEFAULT_VERSION = "1.0.2-beta.3";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 3e217621fddf002ca051db8434e5eab26db7245f Mon Sep 17 00:00:00 2001
From: Matt Naides <63469689+sfc-gh-mnaides@users.noreply.github.com>
Date: Tue, 2 Aug 2022 15:07:03 -0700
Subject: [PATCH 051/356] Run streaming ingest integration tests on build
 (#207)

Run StreamingIngestIT on build
---
 pom.xml                                       |  5 ---
 .../java/net/snowflake/ingest/TestUtils.java  | 31 ++++++++++++++++---
 .../streaming/internal/StreamingIngestIT.java | 27 ++++------------
 3 files changed, 32 insertions(+), 31 deletions(-)

diff --git a/pom.xml b/pom.xml
index b4672607c..8feb90969 100644
--- a/pom.xml
+++ b/pom.xml
@@ -181,11 +181,6 @@
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-failsafe-plugin</artifactId>
                     <version>3.0.0-M5</version>
-                    <configuration>
-                        <excludes>
-                            <exclude>**/StreamingIngestIT.java</exclude>
-                        </excludes>
-                    </configuration>
                 </plugin>
             </plugins>
         </pluginManagement>
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 7f26721fa..95827d5df 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -73,7 +73,11 @@ public class TestUtils {
 
   private static int port = 0;
 
-  private static Connection conn = null;
+  // Keep separate test connections for snowpipe and snowpipe streaming so that session state is
+  // isolated
+  private static Connection snowpipeConn = null;
+
+  private static Connection streamingConn = null;
 
   private static String dummyUser = "user";
   private static int dummyPort = 443;
@@ -206,7 +210,20 @@ private static byte getBlobFormatVersion() {
    * @throws Exception
    */
   public static Connection getConnection() throws Exception {
-    if (conn != null) return conn;
+    return getConnection(false);
+  }
+
+  /**
+   * Create snowflake jdbc connection
+   *
+   * @param isStreamingConnection: is true will return a separate connection for streaming ingest
+   *     tests
+   * @return jdbc connection
+   * @throws Exception
+   */
+  public static Connection getConnection(boolean isStreamingConnection) throws Exception {
+    if (!isStreamingConnection && snowpipeConn != null) return snowpipeConn;
+    if (isStreamingConnection && streamingConn != null) return streamingConn;
 
     if (profile == null) init();
     // check first to see if we have the Snowflake JDBC
@@ -223,10 +240,14 @@ public static Connection getConnection() throws Exception {
     props.put("client_session_keep_alive", "true");
     props.put("privateKey", privateKey);
 
-    conn = DriverManager.getConnection(connectString, props);
-
+    if (isStreamingConnection) {
+      streamingConn = DriverManager.getConnection(connectString, props);
+      // fire off the connection
+      return streamingConn;
+    }
+    snowpipeConn = DriverManager.getConnection(connectString, props);
     // fire off the connection
-    return conn;
+    return snowpipeConn;
   }
 
   /**
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 233bca1bc..37d28bf7e 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -2,6 +2,7 @@
 
 import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
 import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
+import static net.snowflake.ingest.utils.Constants.ROLE;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -55,7 +56,7 @@ public class StreamingIngestIT {
   @Before
   public void beforeAll() throws Exception {
     // Create a streaming ingest client
-    jdbcConnection = TestUtils.getConnection();
+    jdbcConnection = TestUtils.getConnection(true);
     jdbcConnection
         .createStatement()
         .execute(String.format("create or replace database %s;", TEST_DB));
@@ -65,20 +66,6 @@ public void beforeAll() throws Exception {
     jdbcConnection
         .createStatement()
         .execute(String.format("create or replace table %s (c1 char(10));", TEST_TABLE));
-    jdbcConnection
-        .createStatement()
-        .execute("alter session set ENABLE_PR_37692_MULTI_FORMAT_SCANSET=true;");
-    jdbcConnection
-        .createStatement()
-        .execute(
-            String.format(
-                "alter database %s set ENABLE_PR_37692_MULTI_FORMAT_SCANSET=true;", TEST_DB));
-    jdbcConnection
-        .createStatement()
-        .execute(
-            String.format(
-                "alter table %s set ENABLE_PR_37692_MULTI_FORMAT_SCANSET=true;", TEST_TABLE));
-    jdbcConnection.createStatement().execute("alter session set ENABLE_UNIFIED_TABLE_SCAN=true;");
     // Set timezone to UTC
     jdbcConnection.createStatement().execute("alter session set timezone = 'UTC';");
     jdbcConnection
@@ -86,6 +73,9 @@ public void beforeAll() throws Exception {
         .execute(String.format("use warehouse %s", TestUtils.getWarehouse()));
 
     prop = TestUtils.getProperties();
+    if (prop.getProperty(ROLE).equals("DEFAULT_ROLE")) {
+      prop.setProperty(ROLE, "ACCOUNTADMIN");
+    }
     client =
         (SnowflakeStreamingIngestClientInternal)
             SnowflakeStreamingIngestClientFactory.builder("client1").setProperties(prop).build();
@@ -385,11 +375,6 @@ public void testTimeColumnIngest() throws Exception {
                     + " TIMESTAMP_TZ(3), tbig TIME(9), tntzbig TIMESTAMP_NTZ(9), ttzbig"
                     + " TIMESTAMP_TZ(9) );",
                 timeTableName));
-    jdbcConnection
-        .createStatement()
-        .execute(
-            String.format(
-                "alter table %s set ENABLE_PR_37692_MULTI_FORMAT_SCANSET=true;", timeTableName));
     OpenChannelRequest request1 =
         OpenChannelRequest.builder("CHANNEL_TIME")
             .setDBName(TEST_DB)
@@ -642,7 +627,7 @@ public void testMultiThread() throws Exception {
         .execute(
             String.format("create or replace table %s (numcol NUMBER(10,2));", multiThreadTable));
     int numThreads = 20;
-    int numRows = 1000000;
+    int numRows = 10000;
     ExecutorService testThreadPool = Executors.newFixedThreadPool(numThreads);
     CompletableFuture[] futures = new CompletableFuture[numThreads];
     List<SnowflakeStreamingIngestChannel> channelList = new ArrayList<>();

From 3e04fc6539d6ed73797652efadd4492681e9075e Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 17 Aug 2022 17:21:55 -0700
Subject: [PATCH 052/356] SNOW-644959: Return all the extra columns in the
 insertRow response (#210)

- return all the extra columns in the insertRow response, this is needed by the KC schematization work since we want to add all the columns at once during schema evolution
- Clean up the HttpConnectionManagerDaemonThread during client close
- Passing the correct double quotes column name to EP
---
 .../streaming/internal/ArrowRowBuffer.java    | 45 ++++++++++++-------
 ...nowflakeStreamingIngestClientInternal.java |  1 +
 .../streaming/internal/RowBufferTest.java     | 38 +++++++++++++---
 3 files changed, 60 insertions(+), 24 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index bfe27cf94..9c5b3a3a7 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -617,7 +617,7 @@ Field buildField(ColumnMetadata column) {
   private String formatColumnName(String columnName) {
     Utils.assertStringNotNullOrEmpty("invalid column name", columnName);
     return (columnName.charAt(0) == '"' && columnName.charAt(columnName.length() - 1) == '"')
-        ? columnName.substring(1, columnName.length() - 1)
+        ? columnName
         : columnName.toUpperCase();
   }
 
@@ -628,29 +628,40 @@ private String formatColumnName(String columnName) {
    * @return the set of input column names
    */
   private Set<String> verifyInputColumns(Map<String, Object> row) {
-    Set<String> inputColumns =
-        row.keySet().stream().map(this::formatColumnName).collect(Collectors.toSet());
+    Map<String, String> inputColNamesMap =
+        row.keySet().stream().collect(Collectors.toMap(this::formatColumnName, value -> value));
+
+    // Check for extra columns in the row
+    List<String> extraCols = new ArrayList<>();
+    for (String columnName : inputColNamesMap.keySet()) {
+      if (!this.fields.containsKey(columnName)) {
+        extraCols.add(inputColNamesMap.get(columnName));
+      }
+    }
+
+    if (!extraCols.isEmpty()) {
+      throw new SFException(
+          ErrorCode.INVALID_ROW,
+          "Extra columns: " + extraCols,
+          "Columns not present in the table shouldn't be specified.");
+    }
 
+    // Check for missing columns in the row
+    List<String> missingCols = new ArrayList<>();
     for (String columnName : this.nonNullableFieldNames) {
-      if (!inputColumns.contains(columnName)) {
-        throw new SFException(
-            ErrorCode.INVALID_ROW,
-            "Missing column: " + columnName,
-            "Values for all non-nullable columns must be specified.");
+      if (!inputColNamesMap.containsKey(columnName)) {
+        missingCols.add(columnName);
       }
     }
 
-    for (String columnName : inputColumns) {
-      Field field = this.fields.get(columnName);
-      if (field == null) {
-        throw new SFException(
-            ErrorCode.INVALID_ROW,
-            "Extra column: " + columnName,
-            "Columns not present in the table shouldn't be specified.");
-      }
+    if (!missingCols.isEmpty()) {
+      throw new SFException(
+          ErrorCode.INVALID_ROW,
+          "Missing columns: " + missingCols,
+          "Values for all non-nullable columns must be specified.");
     }
 
-    return inputColumns;
+    return inputColNamesMap.keySet();
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index f1e55d1dd..086dd0f72 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -598,6 +598,7 @@ public void close() throws Exception {
       if (this.requestBuilder != null) {
         this.requestBuilder.closeResources();
       }
+      HttpUtil.shutdownHttpConnectionManagerDaemonThread();
       Utils.closeAllocator(this.allocator);
     }
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 84e39a8ed..ad8f5475c 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -71,7 +71,7 @@ public void setupRowBuffer() {
     this.rowBufferOnErrorAbort = new ArrowRowBuffer(this.channelOnErrorAbort);
 
     ColumnMetadata colTinyIntCase = new ColumnMetadata();
-    colTinyIntCase.setName("colTinyInt");
+    colTinyIntCase.setName("\"colTinyInt\"");
     colTinyIntCase.setPhysicalType("SB1");
     colTinyIntCase.setNullable(true);
     colTinyIntCase.setLogicalType("FIXED");
@@ -784,7 +784,7 @@ private void testDoubleQuotesColumnNameHelper(SnowflakeStreamingIngestChannelInt
     ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
 
     ColumnMetadata colDoubleQuotes = new ColumnMetadata();
-    colDoubleQuotes.setName("colDoubleQuotes");
+    colDoubleQuotes.setName("\"colDoubleQuotes\"");
     colDoubleQuotes.setPhysicalType("SB16");
     colDoubleQuotes.setNullable(true);
     colDoubleQuotes.setLogicalType("FIXED");
@@ -889,7 +889,7 @@ private void testArrowE2EHelper(ArrowRowBuffer rowBuffer) {
     InsertValidationResponse response = rowBuffer.insertRows(Collections.singletonList(row1), null);
     Assert.assertFalse(response.hasErrors());
 
-    Assert.assertEquals((byte) 10, rowBuffer.vectorsRoot.getVector("colTinyInt").getObject(0));
+    Assert.assertEquals((byte) 10, rowBuffer.vectorsRoot.getVector("\"colTinyInt\"").getObject(0));
     Assert.assertEquals((byte) 1, rowBuffer.vectorsRoot.getVector("COLTINYINT").getObject(0));
     Assert.assertEquals((short) 2, rowBuffer.vectorsRoot.getVector("COLSMALLINT").getObject(0));
     Assert.assertEquals(3, rowBuffer.vectorsRoot.getVector("COLINT").getObject(0));
@@ -1158,11 +1158,11 @@ private void testStatsE2EHelper(ArrowRowBuffer rowBuffer) {
     Map<String, RowBufferStats> columnEpStats = result.getColumnEps();
 
     Assert.assertEquals(
-        BigInteger.valueOf(11), columnEpStats.get("colTinyInt").getCurrentMaxIntValue());
+        BigInteger.valueOf(11), columnEpStats.get("\"colTinyInt\"").getCurrentMaxIntValue());
     Assert.assertEquals(
-        BigInteger.valueOf(10), columnEpStats.get("colTinyInt").getCurrentMinIntValue());
-    Assert.assertEquals(0, columnEpStats.get("colTinyInt").getCurrentNullCount());
-    Assert.assertEquals(-1, columnEpStats.get("colTinyInt").getDistinctValues());
+        BigInteger.valueOf(10), columnEpStats.get("\"colTinyInt\"").getCurrentMinIntValue());
+    Assert.assertEquals(0, columnEpStats.get("\"colTinyInt\"").getCurrentNullCount());
+    Assert.assertEquals(-1, columnEpStats.get("\"colTinyInt\"").getDistinctValues());
 
     Assert.assertEquals(
         BigInteger.valueOf(1), columnEpStats.get("COLTINYINT").getCurrentMaxIntValue());
@@ -1485,6 +1485,30 @@ private void testMissingColumnCheckHelper(SnowflakeStreamingIngestChannelInterna
     }
   }
 
+  @Test
+  public void testExtraColumnsCheck() {
+    ArrowRowBuffer innerBuffer = new ArrowRowBuffer(this.channelOnErrorContinue);
+
+    ColumnMetadata colBoolean = new ColumnMetadata();
+    colBoolean.setName("COLBOOLEAN1");
+    colBoolean.setPhysicalType("SB1");
+    colBoolean.setNullable(false);
+    colBoolean.setLogicalType("BOOLEAN");
+    colBoolean.setScale(0);
+
+    innerBuffer.setupSchema(Collections.singletonList(colBoolean));
+    Map<String, Object> row = new HashMap<>();
+    row.put("COLBOOLEAN1", true);
+    row.put("COLBOOLEAN2", true);
+    row.put("COLBOOLEAN3", true);
+
+    InsertValidationResponse response = innerBuffer.insertRows(Collections.singletonList(row), "1");
+    Assert.assertTrue(response.hasErrors());
+    Assert.assertEquals(
+        ErrorCode.INVALID_ROW.getMessageCode(),
+        response.getInsertErrors().get(0).getException().getVendorCode());
+  }
+
   @Test
   public void testE2EBoolean() {
     testE2EBooleanHelper(this.channelOnErrorContinue);

From c9413d49deb53a2ce288b2d831a6a52788b8138c Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Fri, 19 Aug 2022 12:19:36 -0700
Subject: [PATCH 053/356] SNOW-637927: Check max column length for Binary data
 type (#206)

We need to check the max column length allowed for the binary data type, otherwise the data will be ingested successfully but not queryable.
---
 .../streaming/internal/ArrowRowBuffer.java    |  7 +++-
 .../internal/DataValidationUtil.java          | 32 +++++++++-----
 .../internal/DataValidationUtilTest.java      | 42 ++++++++++++-------
 3 files changed, 55 insertions(+), 26 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 9c5b3a3a7..7df21cb9e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -956,7 +956,12 @@ private float convertRowToArrow(
               break;
             }
           case BINARY:
-            byte[] bytes = DataValidationUtil.validateAndParseBinary(value);
+            String maxLengthString = field.getMetadata().get(COLUMN_BYTE_LENGTH);
+            byte[] bytes =
+                DataValidationUtil.validateAndParseBinary(
+                    value,
+                    Optional.ofNullable(maxLengthString)
+                        .map(s -> DataValidationUtil.validateAndParseInteger(maxLengthString)));
             ((VarBinaryVector) vector).setSafe(curRowIndex, bytes);
             stats.addStrValue(new String(bytes, StandardCharsets.UTF_8));
             rowBufferSize += bytes.length;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 2fec5b4e1..ae200ca8a 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -25,7 +25,8 @@
 
 /** Utility class for parsing and validating inputs based on Snowflake types */
 class DataValidationUtil {
-  static final int MAX_STRING_LENGTH = 16777216;
+  static final int MAX_STRING_LENGTH = 16 * 1024 * 1024;
+  static final int MAX_BINARY_LENGTH = 8 * 1024 * 1024;
   static final BigInteger MAX_BIGINTEGER = BigInteger.valueOf(10).pow(38);
   static final BigInteger MIN_BIGINTEGER =
       BigInteger.valueOf(-1).multiply(BigInteger.valueOf(10).pow(38));
@@ -52,7 +53,7 @@ static String validateAndParseVariant(Object input) {
           ErrorCode.INVALID_ROW,
           input.toString(),
           String.format(
-              "Variant too long. length=%d maxLength=%d", output.length(), MAX_STRING_LENGTH));
+              "Variant too long: length=%d maxLength=%d", output.length(), MAX_STRING_LENGTH));
     }
     return output;
   }
@@ -246,7 +247,7 @@ static TimestampWrapper validateAndParseTimestampTz(Object input, Map<String, St
   }
 
   /**
-   * Validates the input is less than the suppoed maximum allowed string
+   * Validates the input is less than the supposed maximum allowed string
    *
    * @param input Object to validate and parse to String
    * @param maxLengthOptional Maximum allowed length of the output String, if empty then uses
@@ -260,7 +261,7 @@ static String validateAndParseString(Object input, Optional<Integer> maxLengthOp
       throw new SFException(
           ErrorCode.INVALID_ROW,
           input.toString(),
-          String.format("String too long.  length=%d maxLength=%d", output.length(), maxLength));
+          String.format("String too long: length=%d maxLength=%d", output.length(), maxLength));
     }
     return output;
   }
@@ -547,15 +548,26 @@ static int validateAndParseDate(Object input) {
     }
   }
 
-  static byte[] validateAndParseBinary(Object input) {
+  static byte[] validateAndParseBinary(Object input, Optional<Integer> maxLengthOptional) {
+    byte[] output;
     if (input instanceof byte[]) {
-      return (byte[]) input;
+      output = (byte[]) input;
+    } else {
+      try {
+        output = DatatypeConverter.parseHexBinary(input.toString());
+      } catch (IllegalArgumentException e) {
+        throw new SFException(ErrorCode.INVALID_ROW, input, e.getMessage());
+      }
     }
-    try {
-      return DatatypeConverter.parseHexBinary(input.toString());
-    } catch (IllegalArgumentException e) {
-      throw new SFException(ErrorCode.INVALID_ROW, input, e.getMessage());
+
+    int maxLength = maxLengthOptional.orElse(MAX_BINARY_LENGTH);
+    if (output.length > maxLength) {
+      throw new SFException(
+          ErrorCode.INVALID_ROW,
+          input.toString(),
+          String.format("Binary too long: length=%d maxLength=%d", output.length, maxLength));
     }
+    return output;
   }
 
   /**
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 0d79d1bf6..4386773ee 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -473,21 +473,33 @@ public void testGetTimestampInScale() throws Exception {
   }
 
   @Test
-  public void testValidateAndParseBinary() throws Exception {
-    Assert.assertTrue(
-        Arrays.equals(
-            "honk".getBytes(StandardCharsets.UTF_8),
-            DataValidationUtil.validateAndParseBinary("honk".getBytes(StandardCharsets.UTF_8))));
-    Assert.assertTrue(
-        Arrays.equals(
-            DatatypeConverter.parseHexBinary("12"),
-            DataValidationUtil.validateAndParseBinary("12")));
-
-    Assert.assertTrue(
-        Arrays.equals(
-            DatatypeConverter.parseHexBinary("12"), DataValidationUtil.validateAndParseBinary(12)));
-
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBinary, 123);
+  public void testValidateAndParseBinary() {
+    Assert.assertArrayEquals(
+        "honk".getBytes(StandardCharsets.UTF_8),
+        DataValidationUtil.validateAndParseBinary(
+            "honk".getBytes(StandardCharsets.UTF_8), Optional.empty()));
+
+    Assert.assertArrayEquals(
+        DatatypeConverter.parseHexBinary("12"),
+        DataValidationUtil.validateAndParseBinary("12", Optional.empty()));
+
+    Assert.assertArrayEquals(
+        DatatypeConverter.parseHexBinary("12"),
+        DataValidationUtil.validateAndParseBinary(12, Optional.empty()));
+
+    try {
+      DataValidationUtil.validateAndParseBinary("1212", Optional.of(1));
+      Assert.fail("Expected error for Binary too long");
+    } catch (SFException e) {
+      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
+    }
+
+    try {
+      DataValidationUtil.validateAndParseBinary(123, Optional.empty());
+      Assert.fail("Expected error for invalid Binary format");
+    } catch (SFException e) {
+      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
+    }
   }
 
   @Test

From ab58af1f7fb8b79e81f2b325347f0b0e649e4ed4 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 24 Aug 2022 10:39:01 -0700
Subject: [PATCH 054/356] SNOW-649753: Update InsertError to include
 extra/missing column info for KC schema evolution (#212)

Update InsertError to include extra/missing column information for KC schema evolution
---
 .../streaming/InsertValidationResponse.java   | 43 +++++++++++++++++--
 .../streaming/internal/ArrowRowBuffer.java    | 36 +++++++++++-----
 .../streaming/internal/RowBufferTest.java     | 11 +++--
 3 files changed, 72 insertions(+), 18 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/InsertValidationResponse.java b/src/main/java/net/snowflake/ingest/streaming/InsertValidationResponse.java
index f64260864..0a1deae28 100644
--- a/src/main/java/net/snowflake/ingest/streaming/InsertValidationResponse.java
+++ b/src/main/java/net/snowflake/ingest/streaming/InsertValidationResponse.java
@@ -47,16 +47,21 @@ public void addError(InsertError error) {
   /** Wraps the row content and exception when there is a failure */
   public static class InsertError {
     private final Object rowContent;
-    private final SFException exception;
+    private SFException exception;
 
     // Used to map this error row with original row in the insertRows Iterable.
     // i.e the rowIndex can be index 9 in the list of 10 rows.
     // index is 0 based so as to match with incoming Iterable
     private final long rowIndex;
 
-    public InsertError(Object row, SFException exception, long rowIndex) {
+    // List of extra column names in the input row compared with the table schema
+    private List<String> extraColNames;
+
+    // List of missing non-nullable column names in the input row compared with the table schema
+    private List<String> missingNotNullColNames;
+
+    public InsertError(Object row, long rowIndex) {
       this.rowContent = row;
-      this.exception = exception;
       this.rowIndex = rowIndex;
     }
 
@@ -70,6 +75,15 @@ public String getMessage() {
       return this.exception.getMessage();
     }
 
+    /**
+     * Set the insert exception
+     *
+     * @param exception exception encountered during the insert
+     */
+    public void setException(SFException exception) {
+      this.exception = exception;
+    }
+
     /** Get the exception */
     public SFException getException() {
       return this.exception;
@@ -82,5 +96,28 @@ public SFException getException() {
     public long getRowIndex() {
       return rowIndex;
     }
+
+    /** Set the extra column names in the input row compared with the table schema */
+    public void setExtraColNames(List<String> extraColNames) {
+      this.extraColNames = extraColNames;
+    }
+
+    /** Get the list of extra column names in the input row compared with the table schema */
+    public List<String> getExtraColNames() {
+      return extraColNames;
+    }
+
+    /** Set the missing non-nullable column names in the input row compared with the table schema */
+    public void setMissingNotNullColNames(List<String> missingNotNullColNames) {
+      this.missingNotNullColNames = missingNotNullColNames;
+    }
+
+    /**
+     * Get the list of missing non-nullable column names in the input row compared with the table
+     * schema
+     */
+    public List<String> getMissingNotNullColNames() {
+      return missingNotNullColNames;
+    }
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 7df21cb9e..9c15fab1e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -286,17 +286,22 @@ InsertValidationResponse insertRows(Iterable<Map<String, Object>> rows, String o
         // Used to map incoming row(nth row) to InsertError(for nth row) in response
         long rowIndex = 0;
         for (Map<String, Object> row : rows) {
+          InsertValidationResponse.InsertError error =
+              new InsertValidationResponse.InsertError(row, rowIndex);
           try {
-            rowSize += convertRowToArrow(row, this.vectorsRoot, this.rowCount, this.statsMap);
+            Set<String> inputColumnNames = verifyInputColumns(row, error);
+            rowSize +=
+                convertRowToArrow(
+                    row, this.vectorsRoot, this.rowCount, this.statsMap, inputColumnNames);
             this.rowCount++;
             this.bufferSize += rowSize;
           } catch (SFException e) {
-            response.addError(new InsertValidationResponse.InsertError(row, e, rowIndex));
+            error.setException(e);
+            response.addError(error);
           } catch (Throwable e) {
             logger.logWarn("Unexpected error happens during insertRows: {}", e.getMessage());
-            response.addError(
-                new InsertValidationResponse.InsertError(
-                    row, new SFException(e, ErrorCode.INTERNAL_ERROR, e.getMessage()), rowIndex));
+            error.setException(new SFException(e, ErrorCode.INTERNAL_ERROR, e.getMessage()));
+            response.addError(error);
           }
           rowIndex++;
           if (this.rowCount == Integer.MAX_VALUE) {
@@ -308,8 +313,10 @@ InsertValidationResponse insertRows(Iterable<Map<String, Object>> rows, String o
         float tempRowSize = 0F;
         int tempRowCount = 0;
         for (Map<String, Object> row : rows) {
+          Set<String> inputColumnNames = verifyInputColumns(row, null);
           tempRowSize +=
-              convertRowToArrow(row, this.tempVectorsRoot, tempRowCount, this.tempStatsMap);
+              convertRowToArrow(
+                  row, this.tempVectorsRoot, tempRowCount, this.tempStatsMap, inputColumnNames);
           tempRowCount++;
         }
 
@@ -625,9 +632,11 @@ private String formatColumnName(String columnName) {
    * Verify that the input row columns are all valid
    *
    * @param row the input row
+   * @param error the insert error that we return to the customer
    * @return the set of input column names
    */
-  private Set<String> verifyInputColumns(Map<String, Object> row) {
+  private Set<String> verifyInputColumns(
+      Map<String, Object> row, InsertValidationResponse.InsertError error) {
     Map<String, String> inputColNamesMap =
         row.keySet().stream().collect(Collectors.toMap(this::formatColumnName, value -> value));
 
@@ -640,6 +649,9 @@ private Set<String> verifyInputColumns(Map<String, Object> row) {
     }
 
     if (!extraCols.isEmpty()) {
+      if (error != null) {
+        error.setExtraColNames(extraCols);
+      }
       throw new SFException(
           ErrorCode.INVALID_ROW,
           "Extra columns: " + extraCols,
@@ -655,6 +667,9 @@ private Set<String> verifyInputColumns(Map<String, Object> row) {
     }
 
     if (!missingCols.isEmpty()) {
+      if (error != null) {
+        error.setMissingNotNullColNames(missingCols);
+      }
       throw new SFException(
           ErrorCode.INVALID_ROW,
           "Missing columns: " + missingCols,
@@ -671,16 +686,15 @@ private Set<String> verifyInputColumns(Map<String, Object> row) {
    * @param sourceVectors vectors (buffers) that hold the row
    * @param curRowIndex current row index to use
    * @param statsMap column stats map
+   * @param inputColumnNames list of input column names after formatting
    * @return row size
    */
   private float convertRowToArrow(
       Map<String, Object> row,
       VectorSchemaRoot sourceVectors,
       int curRowIndex,
-      Map<String, RowBufferStats> statsMap) {
-    // Verify all the input columns are valid
-    Set<String> inputColumnNames = verifyInputColumns(row);
-
+      Map<String, RowBufferStats> statsMap,
+      Set<String> inputColumnNames) {
     // Insert values to the corresponding arrow buffers
     float rowBufferSize = 0F;
     for (Map.Entry<String, Object> entry : row.entrySet()) {
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index ad8f5475c..3015099e0 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -1473,9 +1473,11 @@ private void testMissingColumnCheckHelper(SnowflakeStreamingIngestChannelInterna
     if (channel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.CONTINUE) {
       response = innerBuffer.insertRows(Collections.singletonList(row2), "2");
       Assert.assertTrue(response.hasErrors());
+      InsertValidationResponse.InsertError error = response.getInsertErrors().get(0);
       Assert.assertEquals(
-          ErrorCode.INVALID_ROW.getMessageCode(),
-          response.getInsertErrors().get(0).getException().getVendorCode());
+          ErrorCode.INVALID_ROW.getMessageCode(), error.getException().getVendorCode());
+      Assert.assertEquals(
+          Collections.singletonList("COLBOOLEAN"), error.getMissingNotNullColNames());
     } else {
       try {
         innerBuffer.insertRows(Collections.singletonList(row2), "2");
@@ -1504,9 +1506,10 @@ public void testExtraColumnsCheck() {
 
     InsertValidationResponse response = innerBuffer.insertRows(Collections.singletonList(row), "1");
     Assert.assertTrue(response.hasErrors());
+    InsertValidationResponse.InsertError error = response.getInsertErrors().get(0);
     Assert.assertEquals(
-        ErrorCode.INVALID_ROW.getMessageCode(),
-        response.getInsertErrors().get(0).getException().getVendorCode());
+        ErrorCode.INVALID_ROW.getMessageCode(), error.getException().getVendorCode());
+    Assert.assertEquals(Arrays.asList("COLBOOLEAN3", "COLBOOLEAN2"), error.getExtraColNames());
   }
 
   @Test

From cd90b5e42a33a913dacee95fadac04a72a394810 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 24 Aug 2022 17:26:45 -0700
Subject: [PATCH 055/356] new release

---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 8feb90969..75969b9bf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
     <!-- Arifact name and version information -->
     <groupId>net.snowflake</groupId>
     <artifactId>snowflake-ingest-sdk</artifactId>
-    <version>1.0.2-beta.3</version>
+    <version>1.0.2-beta.4</version>
     <packaging>jar</packaging>
     <name>Snowflake Ingest SDK</name>
     <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index f4edc66d7..349c08ace 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -127,7 +127,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.0.2-beta.3";
+  public static final String DEFAULT_VERSION = "1.0.2-beta.4";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 2fa764b92f7334b2fa402f9444233668c39a9e98 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 24 Aug 2022 17:38:08 -0700
Subject: [PATCH 056/356] SNOW-644956: fix string data type  in ARRAY (#211)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When an array (or list) of string is passed into the Ingest SDK, the string doesn’t have double quotes around it when the array is dumped into a string, leading to a failed query later on.

Desired behavior:

[“abc”] → “[“abc”]“

[[“abc”], “de”] → “[[“abc”],”de”]“

Current behavior:

[“abc”] → “[abc]“

[[“abc”], “de”] → “[[abc],de]“

The reason is that the toString method is called to make element of the array into string, and the method will not add quotes around the string as we intend. There could also be issues with other data types whose toString method is not implemented to match our interface.
---
 .../internal/DataValidationUtil.java          | 42 +++++++------------
 .../net/snowflake/ingest/SimpleIngestIT.java  |  2 +-
 .../internal/DataValidationUtilTest.java      | 18 ++++----
 3 files changed, 26 insertions(+), 36 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index ae200ca8a..e739e131f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -8,7 +8,6 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.math.BigDecimal;
 import java.math.BigInteger;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -70,36 +69,23 @@ static String validateAndParseArray(Object input) {
     }
 
     String output;
-
     try {
-      if (input.getClass().isArray()) {
-        if (input instanceof int[]) {
-          output = Arrays.toString((int[]) input);
-        } else if (input instanceof long[]) {
-          output = Arrays.toString((long[]) input);
-        } else if (input instanceof short[]) {
-          output = Arrays.toString((short[]) input);
-        } else if (input instanceof double[]) {
-          output = Arrays.toString((double[]) input);
-        } else if (input instanceof float[]) {
-          output = Arrays.toString((float[]) input);
-        } else if (input instanceof byte[]) {
-          output = Arrays.toString((byte[]) input);
-        } else if (input instanceof char[]) {
-          output = Arrays.toString((char[]) input);
-        } else if (input instanceof boolean[]) {
-          output = Arrays.toString((boolean[]) input);
-        } else if (input instanceof Object[]) {
-          output = Arrays.deepToString((Object[]) input);
-        } else {
-          throw new SFException(ErrorCode.INVALID_ROW, input, "Input array type is not supported.");
-        }
-      } else {
-        output = input.toString();
-      }
+      output = objectMapper.writeValueAsString(input);
     } catch (Exception e) {
       throw new SFException(
-          e, ErrorCode.INVALID_ROW, input, "Input column can't be convert to String.");
+          e,
+          ErrorCode.INVALID_ROW,
+          input.toString(),
+          "Input column can't be convert to a valid string");
+    }
+
+    // Throw an exception if the size is too large
+    if (output.length() > MAX_STRING_LENGTH) {
+      throw new SFException(
+          ErrorCode.INVALID_ROW,
+          input.toString(),
+          String.format(
+              "Array too large. length=%d maxLength=%d", output.length(), MAX_STRING_LENGTH));
     }
     return output;
   }
diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
index 77a00bdae..312d127b9 100644
--- a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
@@ -509,7 +509,7 @@ public void testIngestFilesWithClientInfoWithOldClientSequencer() throws Excepti
       assertEquals(offsetToken, clientStatusResponse.getOffsetToken());
     } catch (IngestResponseException ex) {
       Assert.fail(
-          "The insertFiles API should be successful second time after updaing clientSequencer");
+          "The insertFiles API should be successful second time after updating clientSequencer");
     }
   }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 4386773ee..3e082127f 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -275,27 +275,31 @@ public void testValidateAndParseArray() throws Exception {
     }
 
     int[] intArray = new int[] {1, 2, 3};
-    Assert.assertEquals("[1, 2, 3]", DataValidationUtil.validateAndParseArray(intArray));
+    Assert.assertEquals("[1,2,3]", DataValidationUtil.validateAndParseArray(intArray));
+
+    String[] stringArray = new String[] {"a", "b", "c"};
+    Assert.assertEquals(
+        "[\"a\",\"b\",\"c\"]", DataValidationUtil.validateAndParseArray(stringArray));
 
     Object[] objectArray = new Object[] {1, 2, 3};
-    Assert.assertEquals("[1, 2, 3]", DataValidationUtil.validateAndParseArray(objectArray));
+    Assert.assertEquals("[1,2,3]", DataValidationUtil.validateAndParseArray(objectArray));
 
     Object[] ObjectArrayWithNull = new Object[] {1, null, 3};
     Assert.assertEquals(
-        "[1, null, 3]", DataValidationUtil.validateAndParseArray(ObjectArrayWithNull));
+        "[1,null,3]", DataValidationUtil.validateAndParseArray(ObjectArrayWithNull));
 
     Object[][] nestedArray = new Object[][] {{1, 2, 3}, null, {4, 5, 6}};
     Assert.assertEquals(
-        "[[1, 2, 3], null, [4, 5, 6]]", DataValidationUtil.validateAndParseArray(nestedArray));
+        "[[1,2,3],null,[4,5,6]]", DataValidationUtil.validateAndParseArray(nestedArray));
 
     List<Integer> intList = Arrays.asList(1, 2, 3);
-    Assert.assertEquals("[1, 2, 3]", DataValidationUtil.validateAndParseArray(intList));
+    Assert.assertEquals("[1,2,3]", DataValidationUtil.validateAndParseArray(intList));
 
     List<Object> objectList = Arrays.asList(1, 2, 3);
-    Assert.assertEquals("[1, 2, 3]", DataValidationUtil.validateAndParseArray(objectList));
+    Assert.assertEquals("[1,2,3]", DataValidationUtil.validateAndParseArray(objectList));
 
     List<Object> nestedList = Arrays.asList(Arrays.asList(1, 2, 3), 2, 3);
-    Assert.assertEquals("[[1, 2, 3], 2, 3]", DataValidationUtil.validateAndParseArray(nestedList));
+    Assert.assertEquals("[[1,2,3],2,3]", DataValidationUtil.validateAndParseArray(nestedList));
   }
 
   @Test

From 61065d8a000abcf75e89472c0e8e99f374442800 Mon Sep 17 00:00:00 2001
From: sfc-gh-jfan <jiazhen.fan@snowflake.com>
Date: Tue, 6 Sep 2022 13:11:57 -0700
Subject: [PATCH 057/356] CASEC-1924 rm whitesource legacy files

---
 .github/workflows/snyk-issue.yml    |   4 +-
 .github/workflows/snyk-pr.yml       |  12 +--
 deploy.sh                           |   6 +-
 scripts/run_gh_actions.sh           |   5 --
 scripts/run_whitesource_gh.sh       | 113 ----------------------------
 scripts/wss-java-maven-agent.config |  89 ----------------------
 6 files changed, 9 insertions(+), 220 deletions(-)
 delete mode 100644 scripts/run_whitesource_gh.sh
 delete mode 100644 scripts/wss-java-maven-agent.config

diff --git a/.github/workflows/snyk-issue.yml b/.github/workflows/snyk-issue.yml
index 74d6e1aaf..b586554dd 100644
--- a/.github/workflows/snyk-issue.yml
+++ b/.github/workflows/snyk-issue.yml
@@ -7,7 +7,7 @@ on:
 concurrency: snyk-issue
 
 jobs:
-  whitesource:
+  snyk:
     runs-on: ubuntu-latest
     steps:
     - name: checkout action
@@ -22,7 +22,7 @@ jobs:
       uses: ./whitesource-actions/snyk-issue
       with:
         snyk_org: ${{ secrets.SNYK_ORG_ID_PUBLIC_REPO }}
-        snyk_token:  ${{ secrets.SNYK_GITHUB_INTEGRATION_TOKEN }}
+        snyk_token:  ${{ secrets.SNYK_GITHUB_INTEGRATION_TOKEN_PUBLIC_REPO }}
         jira_token: ${{ secrets.JIRA_TOKEN_PUBLIC_REPO }}
       env:
         GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/snyk-pr.yml b/.github/workflows/snyk-pr.yml
index 048b86917..1ef32b622 100644
--- a/.github/workflows/snyk-pr.yml
+++ b/.github/workflows/snyk-pr.yml
@@ -1,31 +1,31 @@
-name: snyk-pr
+name: Snyk PR
 on:
   pull_request:
     branches:
       - master
 jobs:
-  whitesource:
+  snyk:
     runs-on: ubuntu-latest
     if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }}
     steps:
-    - name: checkout
+    - name: Checkout
       uses: actions/checkout@v3
       with:
         ref: ${{ github.event.pull_request.head.ref }}
         fetch-depth: 0
 
-    - name: checkout action
+    - name: Checkout Action
       uses: actions/checkout@v3
       with:
         repository: snowflakedb/whitesource-actions
         token: ${{ secrets.WHITESOURCE_ACTION_TOKEN }}
         path: whitesource-actions
 
-    - name: PR
+    - name: nyk Pull Request Scan Check
       uses: ./whitesource-actions/snyk-pr
       env:
         PR_TITLE: ${{ github.event.pull_request.title }}
       with:
         jira_token: ${{ secrets.JIRA_TOKEN_PUBLIC_REPO }}
         gh_token: ${{ secrets.GITHUB_TOKEN }}
-        amend: false # true if you want the commit to be amended with the JIRA number
+        amend: false 
\ No newline at end of file
diff --git a/deploy.sh b/deploy.sh
index 0df76eedf..d3cd928fc 100755
--- a/deploy.sh
+++ b/deploy.sh
@@ -84,8 +84,4 @@ mvn ${MVN_OPTIONS[@]} \
     -DstagingRepositoryId=$snowflake_repositories \
     -DstagingDescription="Automated Release"
 
-rm $OSSRH_DEPLOY_SETTINGS_XML
-
-#white source
-chmod 755 ./scripts/run_whitesource_gh.sh
-scripts/run_whitesource_gh.sh
+rm $OSSRH_DEPLOY_SETTINGS_XML
\ No newline at end of file
diff --git a/scripts/run_gh_actions.sh b/scripts/run_gh_actions.sh
index 4ceacbe4b..b7a0b632a 100755
--- a/scripts/run_gh_actions.sh
+++ b/scripts/run_gh_actions.sh
@@ -13,8 +13,3 @@ rc=$?
 if [ $rc -ne 0 ] ; then
   echo Could not perform mvn verify with parameters "${PARAMS[@]}", exit code [$rc]; exit $rc
 fi
-
-# run whitesource
-echo ${PWD}
-chmod 755 ./scripts/run_whitesource_gh.sh
-./scripts/run_whitesource_gh.sh
\ No newline at end of file
diff --git a/scripts/run_whitesource_gh.sh b/scripts/run_whitesource_gh.sh
deleted file mode 100644
index 990352f5c..000000000
--- a/scripts/run_whitesource_gh.sh
+++ /dev/null
@@ -1,113 +0,0 @@
-#!/usr/bin/env bash
-#
-# Run whitesource for components which need versioning
-
-# SCAN_DIRECTORIES is a comma-separated list (as a string) of file paths which contain all source code and build artifacts for this project
-THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-CONNECTOR_DIR="$( dirname "${THIS_DIR}")"
-SCAN_DIRECTORIES="${CONNECTOR_DIR}"
-echo "SCAN_DIRECTORIES:"$SCAN_DIRECTORIES
-
-# If your PROD_BRANCH is not master, you can define it here based on the need
-PROD_BRANCH="master"
-
-PRODUCT_NAME="snowflake-ingest-java"
-
-PROJECT_VERSION=${GITHUB_SHA}
-
-BRANCH_OR_PR_NUMBER="$(echo "${GITHUB_REF}" | awk 'BEGIN { FS = "/" } ; { print $3 }')"
-
-echo "PROJECT_VERSION:"$PROJECT_VERSION
-echo "BRANCH_OR_PR_NUMBER:"$BRANCH_OR_PR_NUMBER
-
-# GITHUB_EVENT_NAME should either be 'push', or 'pull_request'
-if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
-    echo "[INFO] Pull Request"
-    export PROJECT_NAME="PR-${BRANCH_OR_PR_NUMBER}"
-elif [[ "${BRANCH_OR_PR_NUMBER}" == "$PROD_BRANCH" ]]; then
-    echo "[INFO] Production branch"
-    export PROJECT_NAME="$PROD_BRANCH"
-else
-    echo "[INFO] Non Production branch. Skipping wss..."
-    export PROJECT_NAME=""
-fi
-
-echo "PROJECT_NAME:"$PROJECT_NAME
-
-[[ -z "$WHITESOURCE_API_KEY" ]] && echo "[WARNING] No WHITESOURCE_API_KEY is set. No WhiteSource scan will occurr." && exit 0
-
-#if [[ -z "${JOB_BASE_NAME}" ]]; then
-#   echo "[ERROR] No JOB_BASE_NAME is set. Run this on Jenkins"
-#   exit 0
-#fi
-
-# Download the latest whitesource unified agent to do the scanning if there is no existing one
-curl -LO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar
-if [[ ! -f "wss-unified-agent.jar" ]]; then
-    echo "failed to download whitesource unified agent"
-    # if you want to fail the build when failing to download whitesource scanning agent, please use exit 1
-    # exit 1
-fi
-
-# whitesource will scan the folder and detect the corresponding configuration
-# configuration file wss-generated-file.config will be generated under ${SCAN_DIRECTORIES}
-# java -jar wss-unified-agent.jar -detect -d ${SCAN_DIRECTORIES}
-# SCAN_CONFIG="${SCAN_DIRECTORIES}/wss-generated-file.config"
-
-# SCAN_CONFIG is the path to your whitesource configuration file
-SCAN_CONFIG="scripts/wss-java-maven-agent.config"
-echo "SCAN_CONFIG:"$SCAN_CONFIG
-
-echo "[INFO] Running wss.sh for ${PRODUCT_NAME}-${PROJECT_NAME} under ${SCAN_DIRECTORIES}"
-
-if [[ "$PROJECT_NAME" == "$PROD_BRANCH" ]]; then
-    echo "PRODUCTION"
-    java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} \
-        -c ${SCAN_CONFIG} \
-        -project ${PROJECT_NAME} \
-        -product ${PRODUCT_NAME} \
-        -d ${SCAN_DIRECTORIES} \
-        -projectVersion ${PROJECT_VERSION} \
-        -offline true
-        ERR=$?
-        if [[ "$ERR" != "254" && "$ERR" != "0" ]]; then
-            echo "failed to run wss for PROJECT_VERSION=${PROJECT_VERSION} in ${PROJECT_VERSION}..."
-            exit 1
-        fi
-    java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} \
-        -c ${SCAN_CONFIG} \
-        -project ${PROJECT_NAME} \
-        -product ${PRODUCT_NAME} \
-        -projectVersion baseline \
-        -requestFiles whitesource/update-request.txt
-        ERR=$?
-        if [[ "$ERR" != "254" && "$ERR" != "0" ]]; then
-            echo "failed to run wss for PROJECT_VERSION=${PROJECT_VERSION} in baseline"
-            exit 1
-        fi
-    java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} \
-        -c ${SCAN_CONFIG} \
-        -project ${PROJECT_NAME} \
-        -product ${PRODUCT_NAME} \
-        -projectVersion ${PROJECT_VERSION} \
-        -requestFiles whitesource/update-request.txt
-        ERR=$?
-        if [[ "$ERR" != "254" && "$ERR" != "0" ]]; then
-            echo "failed to run wss for PROJECT_VERSION=${PROJECT_VERSION} in ${PROJECT_VERSION}"
-            exit 1
-        fi
-elif [[ -n "$PROJECT_NAME" ]]; then
-    echo "PR"
-    java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} \
-        -c ${SCAN_CONFIG} \
-        -project ${PROJECT_NAME} \
-        -product ${PRODUCT_NAME} \
-        -d ${SCAN_DIRECTORIES} \
-        -projectVersion ${PROJECT_VERSION}
-    ERR=$?
-    if [[ "$ERR" != "254" && "$ERR" != "0" ]]; then
-        echo "failed to run wss for PROJECT_VERSION=${PROJECT_VERSION} in ${PROJECT_VERSION}"
-        exit 1
-    fi
-fi
-exit 0
diff --git a/scripts/wss-java-maven-agent.config b/scripts/wss-java-maven-agent.config
deleted file mode 100644
index 37c54929a..000000000
--- a/scripts/wss-java-maven-agent.config
+++ /dev/null
@@ -1,89 +0,0 @@
-###############################################################
-# WhiteSource Unified-Agent configuration file
-###############################################################
-# MAVEN SCAN MODE
-###############################################################
-
-apiKey=
-#userKey is required if WhiteSource administrator has enabled "Enforce user level access" option
-#userKey=
-#requesterEmail=user@provider.com
-
-projectName=
-projectVersion=
-projectToken=
-#projectTag= key:value
-
-productName=
-productVersion=
-productToken=
-
-#projectPerFolder=true
-#projectPerFolderIncludes=
-#projectPerFolderExcludes=
-
-#wss.connectionTimeoutMinutes=60
-wss.url=https://saas.whitesourcesoftware.com/agent
-
-############
-# Policies #
-############
-checkPolicies=true
-forceCheckAllDependencies=true
-forceUpdate=true
-forceUpdate.failBuildOnPolicyViolation=true
-#updateInventory=false
-
-###########
-# General #
-###########
-#offline=false
-#updateType=APPEND
-#ignoreSourceFiles=true
-#scanComment=
-#failErrorLevel=ALL
-#requireKnownSha1=false
-
-#generateProjectDetailsJson=true
-#generateScanReport=true
-#scanReportTimeoutMinutes=10
-#scanReportFilenameFormat=
-
-#analyzeFrameworks=true
-#analyzeFrameworksReference=
-
-#updateEmptyProject=false
-
-#log.files.level=
-#log.files.maxFileSize=
-#log.files.maxFilesCount=
-#log.files.path=
-
-########################################
-# Package Manager Dependency resolvers #
-########################################
-resolveAllDependencies=false
-
-maven.ignoredScopes=test provided
-maven.resolveDependencies=true
-maven.ignoreSourceFiles=false
-maven.aggregateModules=true
-maven.ignorePomModules=false
-maven.runPreStep=false
-maven.ignoreMvnTreeErrors=true
-maven.environmentPath=
-maven.m2RepositoryPath=
-maven.downloadMissingDependencies=false
-maven.additionalArguments=
-maven.projectNameFromDependencyFile=false										 
-
-###########################################################################################
-# Includes/Excludes Glob patterns - Please use only one exclude line and one include line #
-###########################################################################################
-includes=**/*.java **/*.jar
-
-#Exclude file extensions or specific directories by adding **/*.<extension> or **/<excluded_dir>/**
-excludes=**/*sources.jar **/*javadoc.jar **/original-snowflake-ingest-sdk.jar
-
-case.sensitive.glob=false
-followSymbolicLinks=true

From 20eb36055e3c6e0fa6481fb3962e798248123974 Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Wed, 7 Sep 2022 10:14:32 +0000
Subject: [PATCH 058/356] @snow SNOW-657206 SteamingIngestIT: add random suffix
 to DB name to fix concurrent CI runs

---
 .../streaming/internal/StreamingIngestIT.java | 77 ++++++++++---------
 1 file changed, 41 insertions(+), 36 deletions(-)

diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 37d28bf7e..ddd2fafd2 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -18,6 +18,7 @@
 import java.util.Properties;
 import java.util.Random;
 import java.util.TimeZone;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -41,7 +42,7 @@
 /** Example streaming ingest sdk integration test */
 public class StreamingIngestIT {
   private static final String TEST_TABLE = "STREAMING_INGEST_TEST_TABLE";
-  private static final String TEST_DB = "STREAMING_INGEST_TEST_DB";
+  private static final String TEST_DB_PREFIX = "STREAMING_INGEST_TEST_DB";
   private static final String TEST_SCHEMA = "STREAMING_INGEST_TEST_SCHEMA";
 
   private static final String INTERLEAVED_TABLE_PREFIX = "t_interleaved_test_";
@@ -52,20 +53,25 @@ public class StreamingIngestIT {
 
   private SnowflakeStreamingIngestClientInternal client;
   private Connection jdbcConnection;
+  private String testDb;
 
   @Before
   public void beforeAll() throws Exception {
+    testDb = TEST_DB_PREFIX + "_" + UUID.randomUUID().toString().substring(0, 4);
     // Create a streaming ingest client
     jdbcConnection = TestUtils.getConnection(true);
     jdbcConnection
         .createStatement()
-        .execute(String.format("create or replace database %s;", TEST_DB));
+        .execute(String.format("create or replace database %s;", testDb));
     jdbcConnection
         .createStatement()
-        .execute(String.format("create or replace schema %s;", TEST_SCHEMA));
+        .execute(String.format("create or replace schema %s.%s;", testDb, TEST_SCHEMA));
     jdbcConnection
         .createStatement()
-        .execute(String.format("create or replace table %s (c1 char(10));", TEST_TABLE));
+        .execute(
+            String.format(
+                "create or replace table %s.%s.%s (c1 char(10));",
+                testDb, TEST_SCHEMA, TEST_TABLE));
     // Set timezone to UTC
     jdbcConnection.createStatement().execute("alter session set timezone = 'UTC';");
     jdbcConnection
@@ -84,14 +90,14 @@ public void beforeAll() throws Exception {
   @After
   public void afterAll() throws Exception {
     client.close();
-    jdbcConnection.createStatement().execute(String.format("drop database %s", TEST_DB));
+    jdbcConnection.createStatement().execute(String.format("drop database %s", testDb));
   }
 
   @Test
   public void testSimpleIngest() throws Exception {
     OpenChannelRequest request1 =
         OpenChannelRequest.builder("CHANNEL")
-            .setDBName(TEST_DB)
+            .setDBName(testDb)
             .setSchemaName(TEST_SCHEMA)
             .setTableName(TEST_TABLE)
             .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
@@ -116,7 +122,7 @@ public void testSimpleIngest() throws Exception {
                 .createStatement()
                 .executeQuery(
                     String.format(
-                        "select count(*) from %s.%s.%s", TEST_DB, TEST_SCHEMA, TEST_TABLE));
+                        "select count(*) from %s.%s.%s", testDb, TEST_SCHEMA, TEST_TABLE));
         result.next();
         Assert.assertEquals(1000, result.getLong(1));
 
@@ -126,7 +132,7 @@ public void testSimpleIngest() throws Exception {
                 .executeQuery(
                     String.format(
                         "select * from %s.%s.%s order by c1 limit 2",
-                        TEST_DB, TEST_SCHEMA, TEST_TABLE));
+                        testDb, TEST_SCHEMA, TEST_TABLE));
         result2.next();
         Assert.assertEquals("0", result2.getString(1));
         result2.next();
@@ -169,7 +175,7 @@ public void testParameterOverrides() throws Exception {
 
     OpenChannelRequest request1 =
         OpenChannelRequest.builder("CHANNEL")
-            .setDBName(TEST_DB)
+            .setDBName(testDb)
             .setSchemaName(TEST_SCHEMA)
             .setTableName(TEST_TABLE)
             .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
@@ -194,7 +200,7 @@ public void testParameterOverrides() throws Exception {
                 .createStatement()
                 .executeQuery(
                     String.format(
-                        "select count(*) from %s.%s.%s", TEST_DB, TEST_SCHEMA, TEST_TABLE));
+                        "select count(*) from %s.%s.%s", testDb, TEST_SCHEMA, TEST_TABLE));
         result.next();
         Assert.assertEquals(10, result.getLong(1));
         return;
@@ -269,7 +275,7 @@ public void testCollation() throws Exception {
 
     OpenChannelRequest request1 =
         OpenChannelRequest.builder("CHANNEL")
-            .setDBName(TEST_DB)
+            .setDBName(testDb)
             .setSchemaName(TEST_SCHEMA)
             .setTableName(collationTable)
             .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
@@ -297,7 +303,7 @@ public void testCollation() throws Exception {
                 .executeQuery(
                     String.format(
                         "select min(col), min(noncol) from %s.%s.%s",
-                        TEST_DB, TEST_SCHEMA, collationTable));
+                        testDb, TEST_SCHEMA, collationTable));
         result.next();
         Assert.assertEquals("a", result.getString(1));
         Assert.assertEquals("AA", result.getString(2));
@@ -318,7 +324,7 @@ public void testDecimalColumnIngest() throws Exception {
                 "create or replace table %s (tinyfloat NUMBER(38,2));", decimalTableName));
     OpenChannelRequest request1 =
         OpenChannelRequest.builder("CHANNEL_DECI")
-            .setDBName(TEST_DB)
+            .setDBName(testDb)
             .setSchemaName(TEST_SCHEMA)
             .setTableName(decimalTableName)
             .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
@@ -346,8 +352,7 @@ public void testDecimalColumnIngest() throws Exception {
             jdbcConnection
                 .createStatement()
                 .executeQuery(
-                    String.format(
-                        "select * from %s.%s.%s", TEST_DB, TEST_SCHEMA, decimalTableName));
+                    String.format("select * from %s.%s.%s", testDb, TEST_SCHEMA, decimalTableName));
 
         result.next();
         Assert.assertEquals(-1.1, result.getFloat("TINYFLOAT"), 0.001);
@@ -377,7 +382,7 @@ public void testTimeColumnIngest() throws Exception {
                 timeTableName));
     OpenChannelRequest request1 =
         OpenChannelRequest.builder("CHANNEL_TIME")
-            .setDBName(TEST_DB)
+            .setDBName(testDb)
             .setSchemaName(TEST_SCHEMA)
             .setTableName(timeTableName)
             .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
@@ -422,7 +427,7 @@ public void testTimeColumnIngest() throws Exception {
             jdbcConnection
                 .createStatement()
                 .executeQuery(
-                    String.format("select * from %s.%s.%s", TEST_DB, TEST_SCHEMA, timeTableName));
+                    String.format("select * from %s.%s.%s", testDb, TEST_SCHEMA, timeTableName));
 
         result.next();
         Assert.assertEquals(1609473600123l, result.getTimestamp("TTZSMALL").getTime());
@@ -448,7 +453,7 @@ public void testTimeColumnIngest() throws Exception {
                             + "max(tntzsmall) as mtntzsmall,"
                             + "max(tntzbig) as mtntzbig"
                             + " from %s.%s.%s",
-                        TEST_DB, TEST_SCHEMA, timeTableName));
+                        testDb, TEST_SCHEMA, timeTableName));
 
         result.next();
         Assert.assertEquals(1609473600123L, result.getTimestamp("MTTZSMALL").getTime());
@@ -481,7 +486,7 @@ public void testMultiColumnIngest() throws Exception {
                 multiTableName));
     OpenChannelRequest request1 =
         OpenChannelRequest.builder("CHANNEL_MULTI")
-            .setDBName(TEST_DB)
+            .setDBName(testDb)
             .setSchemaName(TEST_SCHEMA)
             .setTableName(multiTableName)
             .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
@@ -514,7 +519,7 @@ public void testMultiColumnIngest() throws Exception {
             jdbcConnection
                 .createStatement()
                 .executeQuery(
-                    String.format("select * from %s.%s.%s", TEST_DB, TEST_SCHEMA, multiTableName));
+                    String.format("select * from %s.%s.%s", testDb, TEST_SCHEMA, multiTableName));
 
         result.next();
         Assert.assertEquals("honk", result.getString("S"));
@@ -542,7 +547,7 @@ public void testNullableColumns() throws Exception {
                 "create or replace table %s (s text, notnull text NOT NULL);", multiTableName));
     OpenChannelRequest request1 =
         OpenChannelRequest.builder("CHANNEL_MULTI1")
-            .setDBName(TEST_DB)
+            .setDBName(testDb)
             .setSchemaName(TEST_SCHEMA)
             .setTableName(multiTableName)
             .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
@@ -553,7 +558,7 @@ public void testNullableColumns() throws Exception {
 
     OpenChannelRequest request2 =
         OpenChannelRequest.builder("CHANNEL_MULTI2")
-            .setDBName(TEST_DB)
+            .setDBName(testDb)
             .setSchemaName(TEST_SCHEMA)
             .setTableName(multiTableName)
             .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
@@ -594,7 +599,7 @@ public void testNullableColumns() throws Exception {
                 .executeQuery(
                     String.format(
                         "select * from %s.%s.%s order by notnull",
-                        TEST_DB, TEST_SCHEMA, multiTableName));
+                        testDb, TEST_SCHEMA, multiTableName));
 
         result.next();
         Assert.assertEquals("honk", result.getString("S"));
@@ -638,7 +643,7 @@ public void testMultiThread() throws Exception {
               () -> {
                 OpenChannelRequest request =
                     OpenChannelRequest.builder(channelName)
-                        .setDBName(TEST_DB)
+                        .setDBName(testDb)
                         .setSchemaName(TEST_SCHEMA)
                         .setTableName(multiThreadTable)
                         .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
@@ -673,7 +678,7 @@ public void testMultiThread() throws Exception {
                 .executeQuery(
                     String.format(
                         "select count(*), max(numcol), min(numcol) from %s.%s.%s",
-                        TEST_DB, TEST_SCHEMA, multiThreadTable));
+                        testDb, TEST_SCHEMA, multiThreadTable));
 
         result.next();
         Assert.assertEquals(numRows * numThreads, result.getInt(1));
@@ -703,7 +708,7 @@ public void testTwoClientsOneChannel() throws Exception {
 
     OpenChannelRequest requestA =
         OpenChannelRequest.builder("CHANNEL")
-            .setDBName(TEST_DB)
+            .setDBName(testDb)
             .setSchemaName(TEST_SCHEMA)
             .setTableName(TEST_TABLE)
             .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
@@ -771,7 +776,7 @@ public void testTwoClientsOneChannel() throws Exception {
                 .createStatement()
                 .executeQuery(
                     String.format(
-                        "select count(*) from %s.%s.%s", TEST_DB, TEST_SCHEMA, TEST_TABLE));
+                        "select count(*) from %s.%s.%s", testDb, TEST_SCHEMA, TEST_TABLE));
         result.next();
         Assert.assertEquals(4, result.getLong(1));
 
@@ -780,7 +785,7 @@ public void testTwoClientsOneChannel() throws Exception {
                 .createStatement()
                 .executeQuery(
                     String.format(
-                        "select * from %s.%s.%s order by c1", TEST_DB, TEST_SCHEMA, TEST_TABLE));
+                        "select * from %s.%s.%s order by c1", testDb, TEST_SCHEMA, TEST_TABLE));
         result2.next();
         Assert.assertEquals("1", result2.getString(1));
         result2.next();
@@ -804,7 +809,7 @@ public void testAbortOnErrorOption() throws Exception {
 
     OpenChannelRequest request =
         OpenChannelRequest.builder("CHANNEL")
-            .setDBName(TEST_DB)
+            .setDBName(testDb)
             .setSchemaName(TEST_SCHEMA)
             .setTableName(onErrorOptionTable)
             .setOnErrorOption(OpenChannelRequest.OnErrorOption.ABORT)
@@ -848,7 +853,7 @@ public void testAbortOnErrorOption() throws Exception {
                 .executeQuery(
                     String.format(
                         "select count(c1), min(c1), max(c1) from %s.%s.%s",
-                        TEST_DB, TEST_SCHEMA, onErrorOptionTable));
+                        testDb, TEST_SCHEMA, onErrorOptionTable));
         result.next();
         Assert.assertEquals("3", result.getString(1));
         Assert.assertEquals("1", result.getString(2));
@@ -864,7 +869,7 @@ public void testAbortOnErrorOption() throws Exception {
   public void testChannelClose() throws Exception {
     OpenChannelRequest request1 =
         OpenChannelRequest.builder("CHANNEL")
-            .setDBName(TEST_DB)
+            .setDBName(testDb)
             .setSchemaName(TEST_SCHEMA)
             .setTableName(TEST_TABLE)
             .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
@@ -896,7 +901,7 @@ public void testNullValuesOnMultiDataTypes() throws Exception {
 
     OpenChannelRequest request =
         OpenChannelRequest.builder("CHANNEL")
-            .setDBName(TEST_DB)
+            .setDBName(testDb)
             .setSchemaName(TEST_SCHEMA)
             .setTableName(nullValuesOnMultiDataTypesTable)
             .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
@@ -931,7 +936,7 @@ public void testNullValuesOnMultiDataTypes() throws Exception {
                 String.format(
                     "select top 1 c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14,"
                         + " c15, c16, c17 from %s.%s.%s",
-                    TEST_DB, TEST_SCHEMA, nullValuesOnMultiDataTypesTable));
+                    testDb, TEST_SCHEMA, nullValuesOnMultiDataTypesTable));
     result.next();
     for (int idx = 1; idx < 18; idx++) {
       Assert.assertNull(result.getObject(idx));
@@ -960,7 +965,7 @@ private void createTableForInterleavedTest(String tableName) {
   private SnowflakeStreamingIngestChannel openChannel(String tableName, String channelName) {
     OpenChannelRequest request =
         OpenChannelRequest.builder(channelName)
-            .setDBName(TEST_DB)
+            .setDBName(testDb)
             .setSchemaName(TEST_SCHEMA)
             .setTableName(tableName)
             .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
@@ -1009,7 +1014,7 @@ private void verifyInterleavedResult(int rowNumber, String tableName, String str
               .executeQuery(
                   String.format(
                       "select * from %s.%s.%s where str ilike '%s%%' order by num",
-                      TEST_DB, TEST_SCHEMA, tableName, strPrefix));
+                      testDb, TEST_SCHEMA, tableName, strPrefix));
       for (int val = 0; val < rowNumber; val++) {
         result.next();
         Assert.assertEquals(val, result.getLong("NUM"));
@@ -1026,7 +1031,7 @@ private void verifyTableRowCount(int rowNumber, String tableName) {
           jdbcConnection
               .createStatement()
               .executeQuery(
-                  String.format("select count(*) from %s.%s.%s", TEST_DB, TEST_SCHEMA, tableName));
+                  String.format("select count(*) from %s.%s.%s", testDb, TEST_SCHEMA, tableName));
       resultCount.next();
       Assert.assertEquals(rowNumber, resultCount.getLong(1));
     } catch (SQLException e) {

From f53bf93b72795d2c628e5d4603a672a2408f25b8 Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Wed, 27 Jul 2022 14:17:02 +0000
Subject: [PATCH 059/356] @snow SNOW-633432 Refactor RowBuffer and Flusher API

---
 ...SnowflakeStreamingIngestClientFactory.java |   2 +-
 .../streaming/internal/AbstractRowBuffer.java | 460 ++++++++++++++++++
 .../streaming/internal/ArrowFlusher.java      | 135 +++++
 .../streaming/internal/ArrowRowBuffer.java    | 427 ++++------------
 .../streaming/internal/BlobBuilder.java       |  11 +-
 .../streaming/internal/BlobMetadata.java      |   6 +-
 .../streaming/internal/ChannelCache.java      |  23 +-
 .../streaming/internal/ChannelData.java       |  24 +-
 .../internal/DataValidationUtil.java          |  29 +-
 .../streaming/internal/FlushService.java      | 174 ++-----
 .../ingest/streaming/internal/Flusher.java    |  47 ++
 .../streaming/internal/RegisterService.java   |  20 +-
 .../ingest/streaming/internal/RowBuffer.java  |  64 +++
 ...nowflakeStreamingIngestChannelFactory.java |  34 +-
 ...owflakeStreamingIngestChannelInternal.java |  53 +-
 ...nowflakeStreamingIngestClientInternal.java |  39 +-
 .../net/snowflake/ingest/utils/Constants.java |  10 +-
 .../ingest/utils/ParameterProvider.java       |  10 +-
 .../streaming/internal/ChannelCacheTest.java  |  59 ++-
 .../streaming/internal/FlushServiceTest.java  |  81 +--
 .../internal/RegisterServiceTest.java         |  53 +-
 .../streaming/internal/RowBufferTest.java     |  75 +--
 .../SnowflakeStreamingIngestChannelTest.java  |   7 +-
 .../SnowflakeStreamingIngestClientTest.java   |  73 +--
 .../streaming/internal/StreamingIngestIT.java |  16 +-
 25 files changed, 1200 insertions(+), 732 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java
index 9d06ab86e..2af794cd2 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java
@@ -49,7 +49,7 @@ public SnowflakeStreamingIngestClient build() {
       Properties prop = Utils.createProperties(this.prop);
       SnowflakeURL accountURL = new SnowflakeURL(prop.getProperty(Constants.ACCOUNT_URL));
 
-      return new SnowflakeStreamingIngestClientInternal(
+      return new SnowflakeStreamingIngestClientInternal<>(
           this.name, accountURL, prop, this.parameterOverrides);
     }
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
new file mode 100644
index 000000000..3a81dd375
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -0,0 +1,460 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+import net.snowflake.ingest.streaming.InsertValidationResponse;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.util.VisibleForTesting;
+
+/**
+ * The abstract implementation of the buffer in the Streaming Ingest channel that holds the
+ * un-flushed rows, these rows will be converted to the underlying format implementation for faster
+ * processing
+ *
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot}
+ */
+abstract class AbstractRowBuffer<T> implements RowBuffer<T> {
+  private static final Logging logger = new Logging(AbstractRowBuffer.class);
+  // these types cannot be packed into the data chunk because they are not readable by the server
+  // side scanner
+  private static final int INVALID_SERVER_SIDE_DATA_TYPE_ORDINAL = -1;
+
+  // Snowflake table column logical type
+  enum ColumnLogicalType {
+    ANY,
+    BOOLEAN(1),
+    ROWINDEX,
+    NULL(15),
+    REAL(8),
+    FIXED(2),
+    TEXT(9),
+    CHAR,
+    BINARY(10),
+    DATE(7),
+    TIME(6),
+    TIMESTAMP_LTZ(3),
+    TIMESTAMP_NTZ(4),
+    TIMESTAMP_TZ(5),
+    INTERVAL,
+    RAW,
+    ARRAY(13, true),
+    OBJECT(12, true),
+    VARIANT(11, true),
+    ROW,
+    SEQUENCE,
+    FUNCTION,
+    USER_DEFINED_TYPE,
+    ;
+
+    // ordinal should be in sync with the server side scanner
+    private final int ordinal;
+    // whether it is a composite data type: array, object or variant
+    private final boolean object;
+
+    ColumnLogicalType() {
+      // no valid server side ordinal by default
+      this(INVALID_SERVER_SIDE_DATA_TYPE_ORDINAL);
+    }
+
+    ColumnLogicalType(int ordinal) {
+      this(ordinal, false);
+    }
+
+    ColumnLogicalType(int ordinal, boolean object) {
+      this.ordinal = ordinal;
+      this.object = object;
+    }
+
+    /**
+     * Ordinal to encode the data type for the server side scanner
+     *
+     * <p>currently used for Parquet format
+     */
+    public int getOrdinal() {
+      return ordinal;
+    }
+
+    /** Whether the data type is a composite type: OBJECT, VARIANT, ARRAY. */
+    public boolean isObject() {
+      return object;
+    }
+  }
+
+  // Snowflake table column physical type
+  enum ColumnPhysicalType {
+    ROWINDEX(9),
+    DOUBLE(7),
+    SB1(1),
+    SB2(2),
+    SB4(3),
+    SB8(4),
+    SB16(5),
+    LOB(8),
+    BINARY,
+    ROW(10),
+    ;
+
+    // ordinal should be in sync with the server side scanner
+    private final int ordinal;
+
+    ColumnPhysicalType() {
+      // no valid server side ordinal by default
+      this(INVALID_SERVER_SIDE_DATA_TYPE_ORDINAL);
+    }
+
+    ColumnPhysicalType(int ordinal) {
+      this.ordinal = ordinal;
+    }
+
+    /**
+     * Ordinal to encode the data type for the server side scanner
+     *
+     * <p>currently used for Parquet format
+     */
+    public int getOrdinal() {
+      return ordinal;
+    }
+  }
+
+  // Map the column name to the stats
+  @VisibleForTesting Map<String, RowBufferStats> statsMap;
+
+  // Temp stats map to use until all the rows are validated
+  @VisibleForTesting Map<String, RowBufferStats> tempStatsMap;
+
+  // Lock used to protect the buffers from concurrent read/write
+  private final Lock flushLock;
+
+  // Current row count
+  @VisibleForTesting volatile int rowCount;
+
+  // Current buffer size
+  private volatile float bufferSize;
+
+  // Reference to the Streaming Ingest channel that owns this buffer
+  @VisibleForTesting final SnowflakeStreamingIngestChannelInternal<T> owningChannel;
+
+  // Names of non-nullable columns
+  private final Set<String> nonNullableFieldNames;
+
+  AbstractRowBuffer(SnowflakeStreamingIngestChannelInternal<T> channel) {
+    this.owningChannel = channel;
+    this.nonNullableFieldNames = new HashSet<>();
+    this.flushLock = new ReentrantLock();
+    this.rowCount = 0;
+    this.bufferSize = 0F;
+
+    // Initialize empty stats
+    this.statsMap = new HashMap<>();
+    this.tempStatsMap = new HashMap<>();
+  }
+
+  /**
+   * Adds non-nullable filed name.
+   *
+   * @param nonNullableFieldName non-nullable filed name
+   */
+  void addNonNullableFieldName(String nonNullableFieldName) {
+    nonNullableFieldNames.add(nonNullableFieldName);
+  }
+
+  /**
+   * Get the current buffer size
+   *
+   * @return the current buffer size
+   */
+  @Override
+  public float getSize() {
+    return bufferSize;
+  }
+
+  /**
+   * Verify that the input row columns are all valid.
+   *
+   * <p>Checks that the columns, specified in the row, are present in the table and values for all
+   * non-nullable columns are specified.
+   *
+   * @param row the input row
+   * @param error the insert error that we return to the customer
+   * @return the set of input column names
+   */
+  Set<String> verifyInputColumns(
+      Map<String, Object> row, InsertValidationResponse.InsertError error) {
+    Map<String, String> inputColNamesMap =
+        row.keySet().stream()
+            .collect(Collectors.toMap(AbstractRowBuffer::formatColumnName, value -> value));
+
+    // Check for extra columns in the row
+    List<String> extraCols = new ArrayList<>();
+    for (String columnName : inputColNamesMap.keySet()) {
+      if (!hasColumn(columnName)) {
+        extraCols.add(inputColNamesMap.get(columnName));
+      }
+    }
+
+    if (!extraCols.isEmpty()) {
+      if (error != null) {
+        error.setExtraColNames(extraCols);
+      }
+      throw new SFException(
+          ErrorCode.INVALID_ROW,
+          "Extra columns: " + extraCols,
+          "Columns not present in the table shouldn't be specified.");
+    }
+
+    // Check for missing columns in the row
+    List<String> missingCols = new ArrayList<>();
+    for (String columnName : this.nonNullableFieldNames) {
+      if (!inputColNamesMap.containsKey(columnName)) {
+        missingCols.add(columnName);
+      }
+    }
+
+    if (!missingCols.isEmpty()) {
+      if (error != null) {
+        error.setMissingNotNullColNames(missingCols);
+      }
+      throw new SFException(
+          ErrorCode.INVALID_ROW,
+          "Missing columns: " + missingCols,
+          "Values for all non-nullable columns must be specified.");
+    }
+
+    return inputColNamesMap.keySet();
+  }
+
+  /**
+   * Insert a batch of rows into the row buffer
+   *
+   * @param rows input row
+   * @param offsetToken offset token of the latest row in the batch
+   * @return insert response that possibly contains errors because of insertion failures
+   */
+  @Override
+  public InsertValidationResponse insertRows(
+      Iterable<Map<String, Object>> rows, String offsetToken) {
+    float rowSize = 0F;
+    if (!hasColumns()) {
+      throw new SFException(ErrorCode.INTERNAL_ERROR, "Empty column fields");
+    }
+    InsertValidationResponse response = new InsertValidationResponse();
+    this.flushLock.lock();
+    try {
+      if (this.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.CONTINUE) {
+        // Used to map incoming row(nth row) to InsertError(for nth row) in response
+        long rowIndex = 0;
+        for (Map<String, Object> row : rows) {
+          InsertValidationResponse.InsertError error =
+              new InsertValidationResponse.InsertError(row, rowIndex);
+          try {
+            Set<String> inputColumnNames = verifyInputColumns(row, error);
+            rowSize += addRow(row, this.rowCount, this.statsMap, inputColumnNames);
+            this.rowCount++;
+            this.bufferSize += rowSize;
+          } catch (SFException e) {
+            error.setException(e);
+            response.addError(error);
+          } catch (Throwable e) {
+            logger.logWarn("Unexpected error happens during insertRows: {}", e.getMessage());
+            error.setException(new SFException(e, ErrorCode.INTERNAL_ERROR, e.getMessage()));
+            response.addError(error);
+          }
+          rowIndex++;
+          if (this.rowCount == Integer.MAX_VALUE) {
+            throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
+          }
+        }
+      } else {
+        // If the on_error option is ABORT, simply throw the first exception
+        float tempRowSize = 0F;
+        int tempRowCount = 0;
+        for (Map<String, Object> row : rows) {
+          Set<String> inputColumnNames = verifyInputColumns(row, null);
+          tempRowSize += addTempRow(row, tempRowCount, this.tempStatsMap, inputColumnNames);
+          tempRowCount++;
+        }
+
+        moveTempRowsToActualBuffer(tempRowCount);
+
+        rowSize = tempRowSize;
+        if ((long) this.rowCount + tempRowCount >= Integer.MAX_VALUE) {
+          throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
+        }
+        this.rowCount += tempRowCount;
+        this.bufferSize += rowSize;
+        this.statsMap.forEach(
+            (colName, stats) ->
+                this.statsMap.put(
+                    colName,
+                    RowBufferStats.getCombinedStats(stats, this.tempStatsMap.get(colName))));
+      }
+
+      this.owningChannel.setOffsetToken(offsetToken);
+      this.owningChannel.collectRowSize(rowSize);
+    } finally {
+      this.tempStatsMap.values().forEach(RowBufferStats::reset);
+      clearTempRows();
+      this.flushLock.unlock();
+    }
+
+    return response;
+  }
+
+  /**
+   * Flush the data in the row buffer by taking the ownership of the old vectors and pass all the
+   * required info back to the flush service to build the blob
+   *
+   * @return A ChannelData object that contains the info needed by the flush service to build a blob
+   */
+  @Override
+  public ChannelData<T> flush() {
+    logger.logDebug("Start get data for channel={}", this.owningChannel.getFullyQualifiedName());
+    if (this.rowCount > 0) {
+      Optional<T> oldData = Optional.empty();
+      int oldRowCount = 0;
+      float oldBufferSize = 0F;
+      long oldRowSequencer = 0;
+      String oldOffsetToken = null;
+      Map<String, RowBufferStats> oldColumnEps = null;
+
+      logger.logDebug(
+          "Arrow buffer flush about to take lock on channel={}",
+          this.owningChannel.getFullyQualifiedName());
+
+      this.flushLock.lock();
+      try {
+        if (this.rowCount > 0) {
+          // Transfer the ownership of the vectors
+          oldData = getSnapshot();
+          oldRowCount = this.rowCount;
+          oldBufferSize = this.bufferSize;
+          oldRowSequencer = this.owningChannel.incrementAndGetRowSequencer();
+          oldOffsetToken = this.owningChannel.getOffsetToken();
+          oldColumnEps = new HashMap<>(this.statsMap);
+          // Reset everything in the buffer once we save all the info
+          reset();
+        }
+      } finally {
+        this.flushLock.unlock();
+      }
+
+      logger.logDebug(
+          "Arrow buffer flush released lock on channel={}, rowCount={}, bufferSize={}",
+          this.owningChannel.getFullyQualifiedName(),
+          rowCount,
+          bufferSize);
+
+      if (oldData.isPresent()) {
+        ChannelData<T> data = new ChannelData<>();
+        data.setVectors(oldData.get());
+        data.setRowCount(oldRowCount);
+        data.setBufferSize(oldBufferSize);
+        data.setChannel(owningChannel);
+        data.setRowSequencer(oldRowSequencer);
+        data.setOffsetToken(oldOffsetToken);
+        data.setColumnEps(oldColumnEps);
+        return data;
+      }
+    }
+    return null;
+  }
+
+  /** Whether row has a column with a given name. */
+  abstract boolean hasColumn(String name);
+
+  /**
+   * Add an input row to the buffer.
+   *
+   * @param row input row
+   * @param curRowIndex current row index to use
+   * @param statsMap column stats map
+   * @param formattedInputColumnNames list of input column names after formatting
+   * @return row size
+   */
+  abstract float addRow(
+      Map<String, Object> row,
+      int curRowIndex,
+      Map<String, RowBufferStats> statsMap,
+      Set<String> formattedInputColumnNames);
+
+  /**
+   * Add an input row to the temporary row buffer.
+   *
+   * <p>The temporary row buffer is used to add rows before validating all of them. Once all the
+   * rows to add have been validated, the temporary row buffer is moved to the actual one.
+   *
+   * @param row input row
+   * @param curRowIndex current row index to use
+   * @param statsMap column stats map
+   * @param formattedInputColumnNames list of input column names after formatting
+   * @return row size
+   */
+  abstract float addTempRow(
+      Map<String, Object> row,
+      int curRowIndex,
+      Map<String, RowBufferStats> statsMap,
+      Set<String> formattedInputColumnNames);
+
+  /** Move rows from the temporary buffer to the current row buffer. */
+  abstract void moveTempRowsToActualBuffer(int tempRowCount);
+
+  /** Clear the temporary row buffer. */
+  abstract void clearTempRows();
+
+  /** If row has any column. */
+  abstract boolean hasColumns();
+
+  /** Reset the variables after each flush. Note that the caller needs to handle synchronization. */
+  void reset() {
+    this.rowCount = 0;
+    this.bufferSize = 0F;
+    this.statsMap.replaceAll(
+        (key, value) -> new RowBufferStats(value.getCollationDefinitionString()));
+  }
+
+  /** Get buffered data snapshot for later flushing. */
+  abstract Optional<T> getSnapshot();
+
+  /** Normalize the column name, given with the inserted row, to send to server side. */
+  static String formatColumnName(String columnName) {
+    Utils.assertStringNotNullOrEmpty("invalid column name", columnName);
+    return (columnName.charAt(0) == '"' && columnName.charAt(columnName.length() - 1) == '"')
+        ? columnName
+        : columnName.toUpperCase();
+  }
+
+  /**
+   * Given a set of col names to stats, build the right ep info
+   *
+   * @param rowCount: count of rows in the given arrow buffer
+   * @param colStats: map of column name to RowBufferStats
+   * @return the EPs built from column stats
+   */
+  static EpInfo buildEpInfoFromStats(long rowCount, Map<String, RowBufferStats> colStats) {
+    EpInfo epInfo = new EpInfo(rowCount, new HashMap<>());
+    for (Map.Entry<String, RowBufferStats> colStat : colStats.entrySet()) {
+      RowBufferStats stat = colStat.getValue();
+      FileColumnProperties dto = new FileColumnProperties(stat);
+
+      String colName = colStat.getKey();
+      epInfo.getColumnEps().put(colName, dto);
+    }
+    return epInfo;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
new file mode 100644
index 000000000..05bfd047d
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.channels.Channels;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.arrow.vector.VectorLoader;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.compression.CompressionUtil;
+import org.apache.arrow.vector.ipc.ArrowStreamWriter;
+import org.apache.arrow.vector.ipc.ArrowWriter;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
+
+/**
+ * Converts {@link ChannelData} buffered in {@link RowBuffer} to the Arrow format for faster
+ * processing.
+ */
+public class ArrowFlusher implements Flusher<VectorSchemaRoot> {
+  private static final Logging logger = new Logging(ArrowFlusher.class);
+
+  private final Constants.BdecVersion bdecVersion;
+
+  public ArrowFlusher(Constants.BdecVersion bdecVersion) {
+    this.bdecVersion = bdecVersion;
+  }
+
+  @Override
+  public Flusher.SerializationResult serialize(
+      List<ChannelData<VectorSchemaRoot>> channelsDataPerTable,
+      ByteArrayOutputStream chunkData,
+      String filePath)
+      throws IOException {
+    List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
+    long rowCount = 0L;
+    VectorSchemaRoot root = null;
+    ArrowWriter arrowWriter = null;
+    VectorLoader loader = null;
+    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> firstChannel = null;
+    Map<String, RowBufferStats> columnEpStatsMapCombined = null;
+
+    try {
+      for (ChannelData<VectorSchemaRoot> data : channelsDataPerTable) {
+        // Create channel metadata
+        ChannelMetadata channelMetadata =
+            ChannelMetadata.builder()
+                .setOwningChannel(data.getChannel())
+                .setRowSequencer(data.getRowSequencer())
+                .setOffsetToken(data.getOffsetToken())
+                .build();
+        // Add channel metadata to the metadata list
+        channelsMetadataList.add(channelMetadata);
+
+        logger.logDebug(
+            "Start building channel={}, rowCount={}, bufferSize={} in blob={}",
+            data.getChannel().getFullyQualifiedName(),
+            data.getRowCount(),
+            data.getBufferSize(),
+            filePath);
+
+        if (root == null) {
+          columnEpStatsMapCombined = data.getColumnEps();
+          root = data.getVectors();
+          arrowWriter =
+              getArrowBatchWriteMode() == Constants.ArrowBatchWriteMode.STREAM
+                  ? new ArrowStreamWriter(root, null, chunkData)
+                  : new ArrowFileWriterWithCompression(
+                      root,
+                      Channels.newChannel(chunkData),
+                      new CustomCompressionCodec(CompressionUtil.CodecType.ZSTD));
+          loader = new VectorLoader(root);
+          firstChannel = data.getChannel();
+          arrowWriter.start();
+        } else {
+          // This method assumes that channelsDataPerTable is grouped by table. We double check
+          // here and throw an error if the assumption is violated
+          if (!data.getChannel()
+              .getFullyQualifiedTableName()
+              .equals(firstChannel.getFullyQualifiedTableName())) {
+            throw new SFException(ErrorCode.INVALID_DATA_IN_CHUNK);
+          }
+
+          columnEpStatsMapCombined =
+              ChannelData.getCombinedColumnStatsMap(columnEpStatsMapCombined, data.getColumnEps());
+
+          VectorUnloader unloader = new VectorUnloader(data.getVectors());
+          ArrowRecordBatch recordBatch = unloader.getRecordBatch();
+          loader.load(recordBatch);
+          recordBatch.close();
+          data.getVectors().close();
+        }
+
+        // Write channel data using the stream writer
+        arrowWriter.writeBatch();
+        rowCount += data.getRowCount();
+
+        logger.logDebug(
+            "Finish building channel={}, rowCount={}, bufferSize={} in blob={}",
+            data.getChannel().getFullyQualifiedName(),
+            data.getRowCount(),
+            data.getBufferSize(),
+            filePath);
+      }
+    } finally {
+      if (arrowWriter != null) {
+        arrowWriter.close();
+        root.close();
+      }
+    }
+    return new Flusher.SerializationResult(
+        channelsMetadataList, columnEpStatsMapCombined, rowCount);
+  }
+
+  private Constants.ArrowBatchWriteMode getArrowBatchWriteMode() {
+    switch (bdecVersion) {
+      case ONE:
+        return Constants.ArrowBatchWriteMode.STREAM;
+      case TWO:
+        return Constants.ArrowBatchWriteMode.FILE;
+      default:
+        throw new SFException(
+            ErrorCode.INTERNAL_ERROR, "Unsupported BLOB_FORMAT_VERSION: " + bdecVersion);
+    }
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 9c15fab1e..b118c94da 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -9,18 +9,14 @@
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.stream.Collectors;
 import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
 import net.snowflake.client.jdbc.internal.snowflake.common.util.Power10;
-import net.snowflake.ingest.streaming.InsertValidationResponse;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.SFException;
@@ -53,49 +49,7 @@
  * The buffer in the Streaming Ingest channel that holds the un-flushed rows, these rows will be
  * converted to Arrow format for faster processing
  */
-class ArrowRowBuffer {
-
-  // Snowflake table column logical type
-  private static enum ColumnLogicalType {
-    ANY,
-    BOOLEAN,
-    ROWINDEX,
-    NULL,
-    REAL,
-    FIXED,
-    TEXT,
-    CHAR,
-    BINARY,
-    DATE,
-    TIME,
-    TIMESTAMP_LTZ,
-    TIMESTAMP_NTZ,
-    TIMESTAMP_TZ,
-    INTERVAL,
-    RAW,
-    ARRAY,
-    OBJECT,
-    VARIANT,
-    ROW,
-    SEQUENCE,
-    FUNCTION,
-    USER_DEFINED_TYPE,
-  }
-
-  // Snowflake table column physical type
-  private static enum ColumnPhysicalType {
-    ROWINDEX,
-    DOUBLE,
-    SB1,
-    SB2,
-    SB4,
-    SB8,
-    SB16,
-    LOB,
-    BINARY,
-    ROW,
-  }
-
+class ArrowRowBuffer extends AbstractRowBuffer<VectorSchemaRoot> {
   private static final Logging logger = new Logging(ArrowRowBuffer.class);
 
   // Constants for column fields
@@ -124,66 +78,18 @@ private static enum ColumnPhysicalType {
   // Map the column name to Arrow column field
   private final Map<String, Field> fields;
 
-  // Map the column name to the stats
-  @VisibleForTesting Map<String, RowBufferStats> statsMap;
-
-  // Temp stats map to use until all the rows are validated
-  @VisibleForTesting Map<String, RowBufferStats> tempStatsMap;
-
-  // Lock used to protect the buffers from concurrent read/write
-  private final Lock flushLock;
-
-  // Current row count
-  @VisibleForTesting volatile int rowCount;
-
   // Allocator used to allocate the buffers
   private final BufferAllocator allocator;
 
-  // Current buffer size
-  private volatile float bufferSize;
-
-  // Reference to the Streaming Ingest channel that owns this buffer
-  @VisibleForTesting final SnowflakeStreamingIngestChannelInternal owningChannel;
-
-  // Names of non-nullable columns
-  private final Set<String> nonNullableFieldNames;
-
-  /**
-   * Given a set of col names to stats, build the right ep info
-   *
-   * @param rowCount: count of rows in the given arrow buffer
-   * @param colStats: map of column name to RowBufferStats
-   * @return the EPs built from column stats
-   */
-  static EpInfo buildEpInfoFromStats(long rowCount, Map<String, RowBufferStats> colStats) {
-    EpInfo epInfo = new EpInfo(rowCount, new HashMap<>());
-    for (Map.Entry<String, RowBufferStats> colStat : colStats.entrySet()) {
-      RowBufferStats stat = colStat.getValue();
-      FileColumnProperties dto = new FileColumnProperties(stat);
-
-      String colName = colStat.getKey();
-      epInfo.getColumnEps().put(colName, dto);
-    }
-    return epInfo;
-  }
-
   /**
    * Construct a ArrowRowBuffer object
    *
-   * @param channel
+   * @param channel client channel
    */
-  ArrowRowBuffer(SnowflakeStreamingIngestChannelInternal channel) {
-    this.owningChannel = channel;
+  ArrowRowBuffer(SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
+    super(channel);
     this.allocator = channel.getAllocator();
     this.fields = new HashMap<>();
-    this.nonNullableFieldNames = new HashSet<>();
-    this.flushLock = new ReentrantLock();
-    this.rowCount = 0;
-    this.bufferSize = 0F;
-
-    // Initialize empty stats
-    this.statsMap = new HashMap<>();
-    this.tempStatsMap = new HashMap<>();
   }
 
   /**
@@ -191,7 +97,8 @@ static EpInfo buildEpInfoFromStats(long rowCount, Map<String, RowBufferStats> co
    *
    * @param columns list of column metadata
    */
-  void setupSchema(List<ColumnMetadata> columns) {
+  @Override
+  public void setupSchema(List<ColumnMetadata> columns) {
     List<FieldVector> vectors = new ArrayList<>();
     List<FieldVector> tempVectors = new ArrayList<>();
 
@@ -199,7 +106,7 @@ void setupSchema(List<ColumnMetadata> columns) {
       Field field = buildField(column);
       FieldVector vector = field.createVector(this.allocator);
       if (!field.isNullable()) {
-        this.nonNullableFieldNames.add(field.getName());
+        addNonNullableFieldName(field.getName());
       }
       this.fields.put(column.getName(), field);
       vectors.add(vector);
@@ -220,7 +127,8 @@ void setupSchema(List<ColumnMetadata> columns) {
    * Close the row buffer and release resources. Note that the caller needs to handle
    * synchronization
    */
-  void close(String name) {
+  @Override
+  public void close(String name) {
     long allocatedBeforeRelease = this.allocator.getAllocatedMemory();
     if (this.vectorsRoot != null) {
       this.vectorsRoot.close();
@@ -249,197 +157,10 @@ void close(String name) {
   }
 
   /** Reset the variables after each flush. Note that the caller needs to handle synchronization */
+  @Override
   void reset() {
+    super.reset();
     this.vectorsRoot.clear();
-    this.rowCount = 0;
-    this.bufferSize = 0F;
-    this.statsMap.replaceAll(
-        (key, value) -> new RowBufferStats(value.getCollationDefinitionString()));
-  }
-
-  /**
-   * Get the current buffer size
-   *
-   * @return the current buffer size
-   */
-  float getSize() {
-    return this.bufferSize;
-  }
-
-  /**
-   * Insert a batch of rows into the row buffer
-   *
-   * @param rows input row
-   * @param offsetToken offset token of the latest row in the batch
-   * @return insert response that possibly contains errors because of insertion failures
-   */
-  InsertValidationResponse insertRows(Iterable<Map<String, Object>> rows, String offsetToken) {
-    float rowSize = 0F;
-    if (this.fields.isEmpty()) {
-      throw new SFException(ErrorCode.INTERNAL_ERROR, "Empty column fields");
-    }
-
-    InsertValidationResponse response = new InsertValidationResponse();
-    this.flushLock.lock();
-    try {
-      if (this.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.CONTINUE) {
-        // Used to map incoming row(nth row) to InsertError(for nth row) in response
-        long rowIndex = 0;
-        for (Map<String, Object> row : rows) {
-          InsertValidationResponse.InsertError error =
-              new InsertValidationResponse.InsertError(row, rowIndex);
-          try {
-            Set<String> inputColumnNames = verifyInputColumns(row, error);
-            rowSize +=
-                convertRowToArrow(
-                    row, this.vectorsRoot, this.rowCount, this.statsMap, inputColumnNames);
-            this.rowCount++;
-            this.bufferSize += rowSize;
-          } catch (SFException e) {
-            error.setException(e);
-            response.addError(error);
-          } catch (Throwable e) {
-            logger.logWarn("Unexpected error happens during insertRows: {}", e.getMessage());
-            error.setException(new SFException(e, ErrorCode.INTERNAL_ERROR, e.getMessage()));
-            response.addError(error);
-          }
-          rowIndex++;
-          if (this.rowCount == Integer.MAX_VALUE) {
-            throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
-          }
-        }
-      } else {
-        // If the on_error option is ABORT, simply throw the first exception
-        float tempRowSize = 0F;
-        int tempRowCount = 0;
-        for (Map<String, Object> row : rows) {
-          Set<String> inputColumnNames = verifyInputColumns(row, null);
-          tempRowSize +=
-              convertRowToArrow(
-                  row, this.tempVectorsRoot, tempRowCount, this.tempStatsMap, inputColumnNames);
-          tempRowCount++;
-        }
-
-        // If all the rows are inserted successfully, transfer the rows from temp vectors to
-        // the final vectors and update the row size and row count
-        // TODO: switch to VectorSchemaRootAppender once it works for all vector types
-        for (Field field : fields.values()) {
-          FieldVector from = this.tempVectorsRoot.getVector(field);
-          FieldVector to = this.vectorsRoot.getVector(field);
-          for (int rowIdx = 0; rowIdx < tempRowCount; rowIdx++) {
-            to.copyFromSafe(rowIdx, this.rowCount + rowIdx, from);
-          }
-        }
-        rowSize = tempRowSize;
-        if ((long) this.rowCount + tempRowCount >= Integer.MAX_VALUE) {
-          throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
-        }
-        this.rowCount += tempRowCount;
-        this.bufferSize += rowSize;
-        this.statsMap.forEach(
-            (colName, stats) -> {
-              this.statsMap.put(
-                  colName, RowBufferStats.getCombinedStats(stats, this.tempStatsMap.get(colName)));
-            });
-      }
-
-      this.owningChannel.setOffsetToken(offsetToken);
-      this.owningChannel.collectRowSize(rowSize);
-    } finally {
-      this.tempStatsMap.values().forEach(RowBufferStats::reset);
-      this.tempVectorsRoot.clear();
-      this.flushLock.unlock();
-    }
-
-    return response;
-  }
-
-  /**
-   * Flush the data in the row buffer by taking the ownership of the old vectors and pass all the
-   * required info back to the flush service to build the blob
-   *
-   * @return A ChannelData object that contains the info needed by the flush service to build a blob
-   */
-  ChannelData flush() {
-    logger.logDebug("Start get data for channel={}", this.owningChannel.getFullyQualifiedName());
-    if (this.rowCount > 0) {
-      List<FieldVector> oldVectors = new ArrayList<>();
-      int oldRowCount = 0;
-      float oldBufferSize = 0F;
-      long oldRowSequencer = 0;
-      String oldOffsetToken = null;
-      Map<String, RowBufferStats> oldColumnEps = null;
-
-      logger.logDebug(
-          "Arrow buffer flush about to take lock on channel={}",
-          this.owningChannel.getFullyQualifiedName());
-
-      this.flushLock.lock();
-      try {
-        if (this.rowCount > 0) {
-          // Transfer the ownership of the vectors
-          for (FieldVector vector : this.vectorsRoot.getFieldVectors()) {
-            vector.setValueCount(this.rowCount);
-            if (vector instanceof DecimalVector) {
-              // DecimalVectors do not transfer FieldType metadata when using
-              // vector.getTransferPair. We need to explicitly create the new vector to transfer to
-              // in order to keep the metadata.
-              ArrowType arrowType =
-                  new ArrowType.Decimal(
-                      ((DecimalVector) vector).getPrecision(),
-                      ((DecimalVector) vector).getScale(),
-                      DECIMAL_BIT_WIDTH);
-              FieldType fieldType =
-                  new FieldType(
-                      vector.getField().isNullable(),
-                      arrowType,
-                      null,
-                      vector.getField().getMetadata());
-              Field f = new Field(vector.getName(), fieldType, null);
-              DecimalVector newVector = new DecimalVector(f, this.allocator);
-              TransferPair t = vector.makeTransferPair(newVector);
-              t.transfer();
-              oldVectors.add((FieldVector) t.getTo());
-            } else {
-              TransferPair t = vector.getTransferPair(this.allocator);
-              t.transfer();
-              oldVectors.add((FieldVector) t.getTo());
-            }
-          }
-
-          oldRowCount = this.rowCount;
-          oldBufferSize = this.bufferSize;
-          oldRowSequencer = this.owningChannel.incrementAndGetRowSequencer();
-          oldOffsetToken = this.owningChannel.getOffsetToken();
-          oldColumnEps = new HashMap<>(this.statsMap);
-          // Reset everything in the buffer once we save all the info
-          reset();
-        }
-      } finally {
-        this.flushLock.unlock();
-      }
-
-      logger.logDebug(
-          "Arrow buffer flush released lock on channel={}, rowCount={}, bufferSize={}",
-          this.owningChannel.getFullyQualifiedName(),
-          rowCount,
-          bufferSize);
-
-      if (!oldVectors.isEmpty()) {
-        ChannelData data = new ChannelData();
-        VectorSchemaRoot vectors = new VectorSchemaRoot(oldVectors);
-        vectors.setRowCount(oldRowCount);
-        data.setVectors(vectors);
-        data.setBufferSize(oldBufferSize);
-        data.setChannel(this.owningChannel);
-        data.setRowSequencer(oldRowSequencer);
-        data.setOffsetToken(oldOffsetToken);
-        data.setColumnEps(oldColumnEps);
-
-        return data;
-      }
-    }
-    return null;
   }
 
   /**
@@ -621,62 +342,85 @@ Field buildField(ColumnMetadata column) {
     return new Field(column.getName(), fieldType, children);
   }
 
-  private String formatColumnName(String columnName) {
-    Utils.assertStringNotNullOrEmpty("invalid column name", columnName);
-    return (columnName.charAt(0) == '"' && columnName.charAt(columnName.length() - 1) == '"')
-        ? columnName
-        : columnName.toUpperCase();
+  @Override
+  void moveTempRowsToActualBuffer(int tempRowCount) {
+    // If all the rows are inserted successfully, transfer the rows from temp vectors to
+    // the final vectors and update the row size and row count
+    // TODO: switch to VectorSchemaRootAppender once it works for all vector types
+    for (Field field : fields.values()) {
+      FieldVector from = this.tempVectorsRoot.getVector(field);
+      FieldVector to = this.vectorsRoot.getVector(field);
+      for (int rowIdx = 0; rowIdx < tempRowCount; rowIdx++) {
+        to.copyFromSafe(rowIdx, this.rowCount + rowIdx, from);
+      }
+    }
   }
 
-  /**
-   * Verify that the input row columns are all valid
-   *
-   * @param row the input row
-   * @param error the insert error that we return to the customer
-   * @return the set of input column names
-   */
-  private Set<String> verifyInputColumns(
-      Map<String, Object> row, InsertValidationResponse.InsertError error) {
-    Map<String, String> inputColNamesMap =
-        row.keySet().stream().collect(Collectors.toMap(this::formatColumnName, value -> value));
+  @Override
+  void clearTempRows() {
+    tempVectorsRoot.clear();
+  }
 
-    // Check for extra columns in the row
-    List<String> extraCols = new ArrayList<>();
-    for (String columnName : inputColNamesMap.keySet()) {
-      if (!this.fields.containsKey(columnName)) {
-        extraCols.add(inputColNamesMap.get(columnName));
-      }
-    }
+  @Override
+  boolean hasColumns() {
+    return !fields.isEmpty();
+  }
 
-    if (!extraCols.isEmpty()) {
-      if (error != null) {
-        error.setExtraColNames(extraCols);
+  @Override
+  Optional<VectorSchemaRoot> getSnapshot() {
+    List<FieldVector> oldVectors = new ArrayList<>();
+    for (FieldVector vector : this.vectorsRoot.getFieldVectors()) {
+      vector.setValueCount(this.rowCount);
+      if (vector instanceof DecimalVector) {
+        // DecimalVectors do not transfer FieldType metadata when using
+        // vector.getTransferPair. We need to explicitly create the new vector to transfer to
+        // in order to keep the metadata.
+        ArrowType arrowType =
+            new ArrowType.Decimal(
+                ((DecimalVector) vector).getPrecision(),
+                ((DecimalVector) vector).getScale(),
+                DECIMAL_BIT_WIDTH);
+        FieldType fieldType =
+            new FieldType(
+                vector.getField().isNullable(), arrowType, null, vector.getField().getMetadata());
+        Field f = new Field(vector.getName(), fieldType, null);
+        DecimalVector newVector = new DecimalVector(f, this.allocator);
+        TransferPair t = vector.makeTransferPair(newVector);
+        t.transfer();
+        oldVectors.add((FieldVector) t.getTo());
+      } else {
+        TransferPair t = vector.getTransferPair(this.allocator);
+        t.transfer();
+        oldVectors.add((FieldVector) t.getTo());
       }
-      throw new SFException(
-          ErrorCode.INVALID_ROW,
-          "Extra columns: " + extraCols,
-          "Columns not present in the table shouldn't be specified.");
     }
+    VectorSchemaRoot root = new VectorSchemaRoot(oldVectors);
+    root.setRowCount(this.rowCount);
+    return oldVectors.isEmpty() ? Optional.empty() : Optional.of(root);
+  }
 
-    // Check for missing columns in the row
-    List<String> missingCols = new ArrayList<>();
-    for (String columnName : this.nonNullableFieldNames) {
-      if (!inputColNamesMap.containsKey(columnName)) {
-        missingCols.add(columnName);
-      }
-    }
+  @Override
+  boolean hasColumn(String name) {
+    return this.fields.get(name) != null;
+  }
 
-    if (!missingCols.isEmpty()) {
-      if (error != null) {
-        error.setMissingNotNullColNames(missingCols);
-      }
-      throw new SFException(
-          ErrorCode.INVALID_ROW,
-          "Missing columns: " + missingCols,
-          "Values for all non-nullable columns must be specified.");
-    }
+  @Override
+  float addRow(
+      Map<String, Object> row,
+      int curRowIndex,
+      Map<String, RowBufferStats> statsMap,
+      Set<String> formattedInputColumnNames) {
+    return convertRowToArrow(row, vectorsRoot, curRowIndex, statsMap, formattedInputColumnNames);
+  }
 
-    return inputColNamesMap.keySet();
+  @Override
+  float addTempRow(
+      Map<String, Object> row,
+      int curRowIndex,
+      Map<String, RowBufferStats> statsMap,
+      Set<String> formattedInputColumnNames) {
+    return convertRowToArrow(
+        row, tempVectorsRoot, curRowIndex, statsMap, formattedInputColumnNames);
   }
 
   /**
@@ -699,7 +443,7 @@ private float convertRowToArrow(
     float rowBufferSize = 0F;
     for (Map.Entry<String, Object> entry : row.entrySet()) {
       rowBufferSize += 0.125; // 1/8 for null value bitmap
-      String columnName = this.formatColumnName(entry.getKey());
+      String columnName = formatColumnName(entry.getKey());
       Object value = entry.getValue();
       Field field = this.fields.get(columnName);
       Utils.assertNotNull("Arrow column field", field);
@@ -1023,4 +767,9 @@ private void insertNull(FieldVector vector, RowBufferStats stats, int curRowInde
     }
     stats.incCurrentNullCount();
   }
+
+  @Override
+  public Flusher<VectorSchemaRoot> createFlusher(Constants.BdecVersion bdecVersion) {
+    return new ArrowFlusher(bdecVersion);
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
index 96a1ef635..6b76dd600 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
@@ -102,16 +102,13 @@ static Pair<byte[], Integer> compress(
    * @param filePath blob file full path
    * @param chunkData uncompressed chunk data
    * @param blockSizeToAlignTo block size to align to for encryption
-   * @param arrowBatchWriteMode Arrow format write mode
+   * @param compress whether to compress the chunk
    * @return padded compressed chunk data, aligned to blockSizeToAlignTo, and actual length of
    *     compressed data before padding at the end
    * @throws IOException
    */
   static Pair<byte[], Integer> compressIfNeededAndPadChunk(
-      String filePath,
-      ByteArrayOutputStream chunkData,
-      int blockSizeToAlignTo,
-      Constants.ArrowBatchWriteMode arrowBatchWriteMode)
+      String filePath, ByteArrayOutputStream chunkData, int blockSizeToAlignTo, boolean compress)
       throws IOException {
     // Encryption needs padding to the ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES
     // to align with decryption on the Snowflake query path starting from this chunk offset.
@@ -119,7 +116,7 @@ static Pair<byte[], Integer> compressIfNeededAndPadChunk(
     // Hence, the actual chunk size is smaller by the padding size.
     // The compression on the Snowflake query path needs the correct size of the compressed
     // data.
-    if (arrowBatchWriteMode == Constants.ArrowBatchWriteMode.STREAM) {
+    if (compress) {
       // Stream write mode does not support column level compression.
       // Compress the chunk data and pad it for encryption.
       return BlobBuilder.compress(filePath, chunkData, blockSizeToAlignTo);
@@ -147,7 +144,7 @@ static byte[] build(
       List<byte[]> chunksDataList,
       long chunksChecksum,
       long chunksDataSize,
-      Constants.BdecVerion bdecVersion)
+      Constants.BdecVersion bdecVersion)
       throws IOException {
     byte[] chunkMetadataListInBytes = MAPPER.writeValueAsBytes(chunksMetadataList);
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index 7199c664d..2818a392a 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -14,7 +14,7 @@
 class BlobMetadata {
   private final String path;
   private final String md5;
-  private final Constants.BdecVerion bdecVersion;
+  private final Constants.BdecVersion bdecVersion;
   private final List<ChunkMetadata> chunks;
 
   BlobMetadata(String path, String md5, List<ChunkMetadata> chunks) {
@@ -22,7 +22,7 @@ class BlobMetadata {
   }
 
   BlobMetadata(
-      String path, String md5, Constants.BdecVerion bdecVersion, List<ChunkMetadata> chunks) {
+      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks) {
     this.path = path;
     this.md5 = md5;
     this.bdecVersion = bdecVersion;
@@ -30,7 +30,7 @@ class BlobMetadata {
   }
 
   @JsonIgnore
-  Constants.BdecVerion getVersion() {
+  Constants.BdecVersion getVersion() {
     return bdecVersion;
   }
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
index bb4f39ec7..42781deec 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
@@ -13,12 +13,14 @@
  * channels belong to the same table will be stored together in order to combine the channel data
  * during flush. The key is a fully qualified table name and the value is a set of channels that
  * belongs to this table
+ *
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
-class ChannelCache {
+class ChannelCache<T> {
   // Cache to hold all the valid channels, the key for the outer map is FullyQualifiedTableName and
   // the key for the inner map is ChannelName
   private ConcurrentHashMap<
-          String, ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal>>
+          String, ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<T>>>
       cache = new ConcurrentHashMap<>();
 
   /**
@@ -26,12 +28,13 @@ class ChannelCache {
    *
    * @param channel
    */
-  void addChannel(SnowflakeStreamingIngestChannelInternal channel) {
-    ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal> channels =
+  void addChannel(SnowflakeStreamingIngestChannelInternal<T> channel) {
+    ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<T>> channels =
         this.cache.computeIfAbsent(
             channel.getFullyQualifiedTableName(), v -> new ConcurrentHashMap<>());
 
-    SnowflakeStreamingIngestChannelInternal oldChannel = channels.put(channel.getName(), channel);
+    SnowflakeStreamingIngestChannelInternal<T> oldChannel =
+        channels.put(channel.getName(), channel);
     // Invalidate old channel if it exits to block new inserts and return error to users earlier
     if (oldChannel != null) {
       oldChannel.invalidate();
@@ -43,7 +46,7 @@ void addChannel(SnowflakeStreamingIngestChannelInternal channel) {
    *
    * @return
    */
-  Iterator<Map.Entry<String, ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal>>>
+  Iterator<Map.Entry<String, ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<T>>>>
       iterator() {
     return this.cache.entrySet().iterator();
   }
@@ -57,11 +60,11 @@ void closeAllChannels() {
 
   /** Remove a channel in the channel cache if the channel sequencer matches */
   // TODO: background cleaner to cleanup old stale channels that are not closed?
-  void removeChannelIfSequencersMatch(SnowflakeStreamingIngestChannelInternal channel) {
+  void removeChannelIfSequencersMatch(SnowflakeStreamingIngestChannelInternal<T> channel) {
     cache.computeIfPresent(
         channel.getFullyQualifiedTableName(),
         (k, v) -> {
-          SnowflakeStreamingIngestChannelInternal channelInCache = v.get(channel.getName());
+          SnowflakeStreamingIngestChannelInternal<T> channelInCache = v.get(channel.getName());
           // We need to compare the channel sequencer in case the old channel was already been
           // removed
           return channelInCache != null
@@ -81,10 +84,10 @@ void invalidateChannelIfSequencersMatch(
       String channelName,
       Long channelSequencer) {
     String fullyQualifiedTableName = String.format("%s.%s.%s", dbName, schemaName, tableName);
-    ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal> channelsMapPerTable =
+    ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<T>> channelsMapPerTable =
         cache.get(fullyQualifiedTableName);
     if (channelsMapPerTable != null) {
-      SnowflakeStreamingIngestChannelInternal channel = channelsMapPerTable.get(channelName);
+      SnowflakeStreamingIngestChannelInternal<T> channel = channelsMapPerTable.get(channelName);
       if (channel != null && channel.getChannelSequencer().equals(channelSequencer)) {
         channel.invalidate();
       }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
index 013075c56..a9bb3cf2f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
@@ -8,18 +8,20 @@
 import java.util.Map;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
-import org.apache.arrow.vector.VectorSchemaRoot;
 
 /**
  * Contains the data and metadata returned for each channel flush, which will be used to build the
  * blob and register blob request
+ *
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot}
  */
-class ChannelData {
+class ChannelData<T> {
   private Long rowSequencer;
   private String offsetToken;
-  private VectorSchemaRoot vectors;
+  private T vectors;
   private float bufferSize;
-  private SnowflakeStreamingIngestChannelInternal channel;
+  private int rowCount;
+  private SnowflakeStreamingIngestChannelInternal<T> channel;
   private Map<String, RowBufferStats> columnEps;
 
   // TODO performance test this vs in place update
@@ -78,16 +80,20 @@ void setOffsetToken(String offsetToken) {
     this.offsetToken = offsetToken;
   }
 
-  VectorSchemaRoot getVectors() {
+  T getVectors() {
     return this.vectors;
   }
 
-  void setVectors(VectorSchemaRoot vectors) {
+  void setVectors(T vectors) {
     this.vectors = vectors;
   }
 
   int getRowCount() {
-    return this.vectors.getRowCount();
+    return this.rowCount;
+  }
+
+  void setRowCount(int rowCount) {
+    this.rowCount = rowCount;
   }
 
   float getBufferSize() {
@@ -98,11 +104,11 @@ void setBufferSize(float bufferSize) {
     this.bufferSize = bufferSize;
   }
 
-  SnowflakeStreamingIngestChannelInternal getChannel() {
+  SnowflakeStreamingIngestChannelInternal<T> getChannel() {
     return this.channel;
   }
 
-  void setChannel(SnowflakeStreamingIngestChannelInternal channel) {
+  void setChannel(SnowflakeStreamingIngestChannelInternal<T> channel) {
     this.channel = channel;
   }
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index e739e131f..f82f9862f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -126,8 +126,21 @@ static String validateAndParseObject(Object input) {
    */
   static TimestampWrapper validateAndParseTimestampNtzSb16(
       Object input, Map<String, String> metadata) {
+    int scale = Integer.parseInt(metadata.get(ArrowRowBuffer.COLUMN_SCALE));
+    return validateAndParseTimestampNtzSb16(input, scale);
+  }
+
+  /**
+   * Validates and parses input for TIMESTAMP_NTZ Snowflake type
+   *
+   * @param input String date in valid format or seconds past the epoch. Accepts fractional seconds
+   *     with precision up to the column's scale
+   * @param scale decimal scale of timestamp 16 byte integer
+   * @return TimestampWrapper with epoch seconds, fractional seconds, and epoch time in the column
+   *     scale
+   */
+  static TimestampWrapper validateAndParseTimestampNtzSb16(Object input, int scale) {
     try {
-      int scale = Integer.parseInt(metadata.get(ArrowRowBuffer.COLUMN_SCALE));
       String valueString = getStringValue(input);
 
       long epoch;
@@ -194,10 +207,22 @@ private static int getFractionFromTimestamp(SFTimestamp input) {
    *     scale
    */
   static TimestampWrapper validateAndParseTimestampTz(Object input, Map<String, String> metadata) {
+    int scale = Integer.parseInt(metadata.get(ArrowRowBuffer.COLUMN_SCALE));
+    return validateAndParseTimestampTz(input, scale);
+  }
+
+  /**
+   * Validates and parses input for TIMESTAMP_TZ Snowflake type
+   *
+   * @param input TIMESTAMP_TZ in "2021-01-01 01:00:00 +0100" format
+   * @param scale decimal scale of timestamp 16 byte integer
+   * @return TimestampWrapper with epoch seconds, fractional seconds, and epoch time in the column
+   *     scale
+   */
+  static TimestampWrapper validateAndParseTimestampTz(Object input, int scale) {
     try {
       if (input instanceof String) {
         String stringInput = (String) input;
-        int scale = Integer.parseInt(metadata.get(ArrowRowBuffer.COLUMN_SCALE));
         SFTimestamp timestamp = snowflakeDateTimeFormatter.parse(stringInput);
         if (timestamp == null) {
           throw new SFException(
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 7d0860bdb..59f221158 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -15,7 +15,6 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
-import java.nio.channels.Channels;
 import java.security.InvalidAlgorithmParameterException;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
@@ -48,13 +47,7 @@
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.Utils;
 import org.apache.arrow.util.VisibleForTesting;
-import org.apache.arrow.vector.VectorLoader;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.VectorUnloader;
-import org.apache.arrow.vector.compression.CompressionUtil;
-import org.apache.arrow.vector.ipc.ArrowStreamWriter;
-import org.apache.arrow.vector.ipc.ArrowWriter;
-import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
 
 /**
  * Responsible for flushing data from client to Snowflake tables. When a flush is triggered, it will
@@ -63,16 +56,18 @@
  * <li>build the blob using the ChannelData returned from previous step
  * <li>upload the blob to stage
  * <li>register the blob to the targeted Snowflake table
+ *
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
-class FlushService {
+class FlushService<T> {
 
   // Static class to save the list of channels that are used to build a blob, which is mainly used
   // to invalidate all the channels when there is a failure
-  static class BlobData {
+  static class BlobData<T> {
     private final String filePath;
-    private final List<List<ChannelData>> data;
+    private final List<List<ChannelData<T>>> data;
 
-    BlobData(String filePath, List<List<ChannelData>> data) {
+    BlobData(String filePath, List<List<ChannelData<T>>> data) {
       this.filePath = filePath;
       this.data = data;
     }
@@ -81,7 +76,7 @@ String getFilePath() {
       return filePath;
     }
 
-    List<List<ChannelData>> getData() {
+    List<List<ChannelData<T>>> getData() {
       return data;
     }
   }
@@ -92,7 +87,7 @@ List<List<ChannelData>> getData() {
   private final AtomicLong counter;
 
   // The client that owns this flush service
-  private final SnowflakeStreamingIngestClientInternal owningClient;
+  private final SnowflakeStreamingIngestClientInternal<T> owningClient;
 
   // Thread to schedule the flush job
   @VisibleForTesting ScheduledExecutorService flushWorker;
@@ -104,13 +99,13 @@ List<List<ChannelData>> getData() {
   @VisibleForTesting ExecutorService buildUploadWorkers;
 
   // Reference to the channel cache
-  private final ChannelCache channelCache;
+  private final ChannelCache<T> channelCache;
 
   // Reference to the Streaming Ingest stage
   private final StreamingIngestStage targetStage;
 
   // Reference to register service
-  private final RegisterService registerService;
+  private final RegisterService<T> registerService;
 
   // Indicates whether we need to schedule a flush
   @VisibleForTesting volatile boolean isNeedFlush;
@@ -125,7 +120,7 @@ List<List<ChannelData>> getData() {
   private final Map<String, Timer.Context> latencyTimerContextMap;
 
   // blob file version
-  private final Constants.BdecVerion bdecVersion;
+  private final Constants.BdecVersion bdecVersion;
 
   /**
    * Constructor for TESTING that takes (usually mocked) StreamingIngestStage
@@ -135,15 +130,15 @@ List<List<ChannelData>> getData() {
    * @param isTestMode
    */
   FlushService(
-      SnowflakeStreamingIngestClientInternal client,
-      ChannelCache cache,
+      SnowflakeStreamingIngestClientInternal<T> client,
+      ChannelCache<T> cache,
       StreamingIngestStage targetStage, // For testing
       boolean isTestMode) {
     this.owningClient = client;
     this.channelCache = cache;
     this.targetStage = targetStage;
     this.counter = new AtomicLong(0);
-    this.registerService = new RegisterService(client, isTestMode);
+    this.registerService = new RegisterService<>(client, isTestMode);
     this.isNeedFlush = false;
     this.lastFlushTime = System.currentTimeMillis();
     this.isTestMode = isTestMode;
@@ -160,7 +155,7 @@ List<List<ChannelData>> getData() {
    * @param isTestMode
    */
   FlushService(
-      SnowflakeStreamingIngestClientInternal client, ChannelCache cache, boolean isTestMode) {
+      SnowflakeStreamingIngestClientInternal<T> client, ChannelCache<T> cache, boolean isTestMode) {
     this.owningClient = client;
     this.channelCache = cache;
     try {
@@ -175,7 +170,7 @@ List<List<ChannelData>> getData() {
       throw new SFException(err, ErrorCode.UNABLE_TO_CONNECT_TO_STAGE);
     }
 
-    this.registerService = new RegisterService(client, isTestMode);
+    this.registerService = new RegisterService<>(client, isTestMode);
     this.counter = new AtomicLong(0);
     this.isNeedFlush = false;
     this.lastFlushTime = System.currentTimeMillis();
@@ -318,23 +313,25 @@ private void createWorkers() {
    * off a build blob work when certain size has reached or we have reached the end
    */
   void distributeFlushTasks() {
-    Iterator<Map.Entry<String, ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal>>>
+    Iterator<
+            Map.Entry<
+                String, ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<T>>>>
         itr = this.channelCache.iterator();
-    List<Pair<BlobData, CompletableFuture<BlobMetadata>>> blobs = new ArrayList<>();
+    List<Pair<BlobData<T>, CompletableFuture<BlobMetadata>>> blobs = new ArrayList<>();
 
     while (itr.hasNext()) {
-      List<List<ChannelData>> blobData = new ArrayList<>();
+      List<List<ChannelData<T>>> blobData = new ArrayList<>();
       float totalBufferSize = 0;
 
       // Distribute work at table level, create a new blob if reaching the blob size limit
       while (itr.hasNext() && totalBufferSize <= MAX_BLOB_SIZE_IN_BYTES) {
-        ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal> table =
+        ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<T>> table =
             itr.next().getValue();
-        List<ChannelData> channelsDataPerTable = new ArrayList<>();
+        List<ChannelData<T>> channelsDataPerTable = new ArrayList<>();
         // TODO: we could do parallel stream to get the channelData if needed
-        for (SnowflakeStreamingIngestChannelInternal channel : table.values()) {
+        for (SnowflakeStreamingIngestChannelInternal<T> channel : table.values()) {
           if (channel.isValid()) {
-            ChannelData data = channel.getData();
+            ChannelData<T> data = channel.getData();
             if (data != null) {
               channelsDataPerTable.add(data);
               totalBufferSize += data.getBufferSize();
@@ -354,7 +351,7 @@ void distributeFlushTasks() {
         }
         blobs.add(
             new Pair<>(
-                new BlobData(filePath, blobData),
+                new BlobData<>(filePath, blobData),
                 CompletableFuture.supplyAsync(
                     () -> {
                       try {
@@ -401,7 +398,7 @@ void distributeFlushTasks() {
    *     belongs to the same table. Will error if this is not the case
    * @return BlobMetadata for FlushService.upload
    */
-  BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
+  BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData)
       throws IOException, NoSuchAlgorithmException, InvalidAlgorithmParameterException,
           NoSuchPaddingException, IllegalBlockSizeException, BadPaddingException,
           InvalidKeyException {
@@ -412,93 +409,22 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
     Timer.Context buildContext = Utils.createTimerContext(this.owningClient.buildLatency);
 
     // TODO: channels with different schema can't be combined even if they belongs to same table
-    for (List<ChannelData> channelsDataPerTable : blobData) {
-      List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
-      long rowCount = 0L;
-      VectorSchemaRoot root = null;
-      ArrowWriter arrowWriter = null;
-      VectorLoader loader = null;
-      SnowflakeStreamingIngestChannelInternal firstChannel = null;
+    for (List<ChannelData<T>> channelsDataPerTable : blobData) {
       ByteArrayOutputStream chunkData = new ByteArrayOutputStream();
-      Map<String, RowBufferStats> columnEpStatsMapCombined = null;
-
-      try {
-        for (ChannelData data : channelsDataPerTable) {
-          // Create channel metadata
-          ChannelMetadata channelMetadata =
-              ChannelMetadata.builder()
-                  .setOwningChannel(data.getChannel())
-                  .setRowSequencer(data.getRowSequencer())
-                  .setOffsetToken(data.getOffsetToken())
-                  .build();
-          // Add channel metadata to the metadata list
-          channelsMetadataList.add(channelMetadata);
-
-          logger.logDebug(
-              "Start building channel={}, rowCount={}, bufferSize={} in blob={}",
-              data.getChannel().getFullyQualifiedName(),
-              data.getRowCount(),
-              data.getBufferSize(),
-              filePath);
-
-          if (root == null) {
-            columnEpStatsMapCombined = data.getColumnEps();
-            root = data.getVectors();
-            arrowWriter =
-                getArrowBatchWriteMode() == Constants.ArrowBatchWriteMode.STREAM
-                    ? new ArrowStreamWriter(root, null, chunkData)
-                    : new ArrowFileWriterWithCompression(
-                        root,
-                        Channels.newChannel(chunkData),
-                        new CustomCompressionCodec(CompressionUtil.CodecType.ZSTD));
-            loader = new VectorLoader(root);
-            firstChannel = data.getChannel();
-            arrowWriter.start();
-          } else {
-            // This method assumes that channelsDataPerTable is grouped by table. We double check
-            // here and throw an error if the assumption is violated
-            if (!data.getChannel()
-                .getFullyQualifiedTableName()
-                .equals(firstChannel.getFullyQualifiedTableName())) {
-              throw new SFException(ErrorCode.INVALID_DATA_IN_CHUNK);
-            }
-
-            columnEpStatsMapCombined =
-                ChannelData.getCombinedColumnStatsMap(
-                    columnEpStatsMapCombined, data.getColumnEps());
-
-            VectorUnloader unloader = new VectorUnloader(data.getVectors());
-            ArrowRecordBatch recordBatch = unloader.getRecordBatch();
-            loader.load(recordBatch);
-            recordBatch.close();
-            data.getVectors().close();
-          }
+      SnowflakeStreamingIngestChannelInternal<T> firstChannel =
+          channelsDataPerTable.get(0).getChannel();
 
-          // Write channel data using the stream writer
-          arrowWriter.writeBatch();
-          rowCount += data.getRowCount();
+      Flusher<T> flusher = firstChannel.getRowBuffer().createFlusher(bdecVersion);
+      Flusher.SerializationResult result =
+          flusher.serialize(channelsDataPerTable, chunkData, filePath);
 
-          logger.logDebug(
-              "Finish building channel={}, rowCount={}, bufferSize={} in blob={}",
-              data.getChannel().getFullyQualifiedName(),
-              data.getRowCount(),
-              data.getBufferSize(),
-              filePath);
-        }
-      } finally {
-        if (arrowWriter != null) {
-          arrowWriter.close();
-          root.close();
-        }
-      }
-
-      if (!channelsMetadataList.isEmpty()) {
+      if (!result.channelsMetadataList.isEmpty()) {
         Pair<byte[], Integer> compressionResult =
             BlobBuilder.compressIfNeededAndPadChunk(
                 filePath,
                 chunkData,
                 Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES,
-                getArrowBatchWriteMode());
+                bdecVersion == Constants.BdecVersion.ONE);
         byte[] compressedAndPaddedChunkData = compressionResult.getFirst();
         int compressedChunkLength = compressionResult.getSecond();
 
@@ -507,6 +433,7 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
         // We need to maintain IV as a block counter for the whole file, even interleaved,
         // to align with decryption on the Snowflake query path.
         // TODO: address alignment for the header SNOW-557866
+        // TODO: encryption is not yet supported by server side for Parquet
         long iv = curDataSize / Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES;
         byte[] encryptedCompressedChunkData =
             Cryptor.encrypt(
@@ -527,10 +454,12 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
                 // The compressedChunkLength is used because it is the actual data size used for
                 // decompression and md5 calculation on server side.
                 .setChunkLength(compressedChunkLength)
-                .setChannelList(channelsMetadataList)
+                .setChannelList(result.channelsMetadataList)
                 .setChunkMD5(md5)
                 .setEncryptionKeyId(firstChannel.getEncryptionKeyId())
-                .setEpInfo(ArrowRowBuffer.buildEpInfoFromStats(rowCount, columnEpStatsMapCombined))
+                .setEpInfo(
+                    AbstractRowBuffer.buildEpInfoFromStats(
+                        result.rowCount, result.columnEpStatsMapCombined))
                 .build();
 
         // Add chunk metadata and data to the list
@@ -542,15 +471,15 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
         logger.logInfo(
             "Finish building chunk in blob={}, table={}, rowCount={}, startOffset={},"
                 + " uncompressedSize={}, compressedChunkLength={}, encryptedCompressedSize={},"
-                + " arrowBatchWriteMode={}",
+                + " bdecVersion={}",
             filePath,
             firstChannel.getFullyQualifiedTableName(),
-            rowCount,
+            result.rowCount,
             startOffset,
             chunkData.size(),
             compressedChunkLength,
             encryptedCompressedChunkDataSize,
-            getArrowBatchWriteMode());
+            bdecVersion);
       }
     }
 
@@ -565,17 +494,6 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData>> blobData)
     return upload(filePath, blob, chunksMetadataList);
   }
 
-  private Constants.ArrowBatchWriteMode getArrowBatchWriteMode() {
-    switch (bdecVersion) {
-      case ONE:
-        return Constants.ArrowBatchWriteMode.STREAM;
-      case TWO:
-        return Constants.ArrowBatchWriteMode.FILE;
-      default:
-        throw new IllegalArgumentException("Unsupported BLOB_FORMAT_VERSION: " + bdecVersion);
-    }
-  }
-
   /**
    * Upload a blob to Streaming Ingest dedicated stage
    *
@@ -683,12 +601,14 @@ String getFilePath(Calendar calendar, String clientPrefix) {
    *
    * @param blobData list of channels that belongs to the blob
    */
-  void invalidateAllChannelsInBlob(List<List<ChannelData>> blobData) {
+  void invalidateAllChannelsInBlob(List<List<ChannelData<T>>> blobData) {
     blobData.forEach(
         chunkData ->
             chunkData.forEach(
                 channelData -> {
-                  channelData.getVectors().close();
+                  if (channelData.getVectors() instanceof VectorSchemaRoot) {
+                    ((VectorSchemaRoot) channelData.getVectors()).close();
+                  }
                   this.owningClient
                       .getChannelCache()
                       .invalidateChannelIfSequencersMatch(
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
new file mode 100644
index 000000000..4ac7acf8c
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Interface to convert {@link ChannelData} buffered in {@link RowBuffer} to the underlying format
+ * implementation for faster processing.
+ *
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ */
+public interface Flusher<T> {
+  /**
+   * Serialize buffered rows into the underlying format.
+   *
+   * @param channelsDataPerTable buffered rows
+   * @param chunkData output
+   * @param filePath file path
+   * @return {@link SerializationResult}
+   * @throws IOException
+   */
+  SerializationResult serialize(
+      List<ChannelData<T>> channelsDataPerTable, ByteArrayOutputStream chunkData, String filePath)
+      throws IOException;
+
+  /** Holds result of the buffered rows conversion: channel metadata and stats. */
+  class SerializationResult {
+    final List<ChannelMetadata> channelsMetadataList;
+    final Map<String, RowBufferStats> columnEpStatsMapCombined;
+    final long rowCount;
+
+    public SerializationResult(
+        List<ChannelMetadata> channelsMetadataList,
+        Map<String, RowBufferStats> columnEpStatsMapCombined,
+        long rowCount) {
+      this.channelsMetadataList = channelsMetadataList;
+      this.columnEpStatsMapCombined = columnEpStatsMapCombined;
+      this.rowCount = rowCount;
+    }
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index 216bf3caa..844852be5 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -23,18 +23,20 @@
 /**
  * Register one or more blobs to the targeted Snowflake table, it will be done using the dedicated
  * thread in order to maintain ordering per channel
+ *
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
-class RegisterService {
+class RegisterService<T> {
 
   private static final Logging logger = new Logging(RegisterService.class);
 
   // Reference to the client that owns this register service
-  private final SnowflakeStreamingIngestClientInternal owningClient;
+  private final SnowflakeStreamingIngestClientInternal<T> owningClient;
 
   // Contains one or more blob metadata that will be registered to Snowflake
   // The key is the BlobData object, and the value is the BlobMetadata future that will complete
   // when the blob is uploaded successfully
-  private final List<Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>>> blobsList;
+  private final List<Pair<FlushService.BlobData<T>, CompletableFuture<BlobMetadata>>> blobsList;
 
   // Lock to protect the read/write access of m_blobsList
   private final Lock blobsListLock;
@@ -59,7 +61,7 @@ class RegisterService {
    *
    * @param blobs
    */
-  void addBlobs(List<Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>>> blobs) {
+  void addBlobs(List<Pair<FlushService.BlobData<T>, CompletableFuture<BlobMetadata>>> blobs) {
     if (!blobs.isEmpty()) {
       this.blobsListLock.lock();
       try {
@@ -77,12 +79,12 @@ void addBlobs(List<Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>>>
    * @param latencyTimerContextMap the map that stores the latency timer for each blob
    * @return a list of blob names that have errors during registration
    */
-  List<FlushService.BlobData> registerBlobs(Map<String, Timer.Context> latencyTimerContextMap) {
-    List<FlushService.BlobData> errorBlobs = new ArrayList<>();
+  List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyTimerContextMap) {
+    List<FlushService.BlobData<T>> errorBlobs = new ArrayList<>();
     if (!this.blobsList.isEmpty()) {
       // Will skip and try again later if someone else is holding the lock
       if (this.blobsListLock.tryLock()) {
-        List<Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>>> oldList = null;
+        List<Pair<FlushService.BlobData<T>, CompletableFuture<BlobMetadata>>> oldList = null;
         try {
           // Create a copy of the list when we have the lock, and then release the lock to unblock
           // other writers while we work on the old list
@@ -113,7 +115,7 @@ List<FlushService.BlobData> registerBlobs(Map<String, Timer.Context> latencyTime
           while (idx < oldList.size()
               && System.currentTimeMillis() - startTime
                   <= TimeUnit.SECONDS.toMillis(BLOB_UPLOAD_TIMEOUT_IN_SEC * 2)) {
-            Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>> futureBlob =
+            Pair<FlushService.BlobData<T>, CompletableFuture<BlobMetadata>> futureBlob =
                 oldList.get(idx);
             try {
               logger.logDebug(
@@ -210,7 +212,7 @@ List<FlushService.BlobData> registerBlobs(Map<String, Timer.Context> latencyTime
    *
    * @return the blobsList
    */
-  List<Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>>> getBlobsList() {
+  List<Pair<FlushService.BlobData<T>, CompletableFuture<BlobMetadata>>> getBlobsList() {
     return this.blobsList;
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
new file mode 100644
index 000000000..a229f4db3
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.util.List;
+import java.util.Map;
+import net.snowflake.ingest.streaming.InsertValidationResponse;
+import net.snowflake.ingest.utils.Constants;
+
+/**
+ * Interface for the buffer in the Streaming Ingest channel that holds the un-flushed rows, these
+ * rows will be converted to the underlying format implementation for faster processing
+ *
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ */
+interface RowBuffer<T> {
+  /**
+   * Setup the column fields and vectors using the column metadata from the server
+   *
+   * @param columns list of column metadata
+   */
+  void setupSchema(List<ColumnMetadata> columns);
+
+  /**
+   * Insert a batch of rows into the row buffer
+   *
+   * @param rows input row
+   * @param offsetToken offset token of the latest row in the batch
+   * @return insert response that possibly contains errors because of insertion failures
+   */
+  InsertValidationResponse insertRows(Iterable<Map<String, Object>> rows, String offsetToken);
+
+  /**
+   * Flush the data in the row buffer by taking the ownership of the old vectors and pass all the
+   * required info back to the flush service to build the blob
+   *
+   * @return A ChannelData object that contains the info needed by the flush service to build a blob
+   */
+  ChannelData<T> flush();
+
+  /**
+   * Close the row buffer and release resources. Note that the caller needs to handle
+   * synchronization
+   */
+  void close(String name);
+
+  /**
+   * Get the current buffer size
+   *
+   * @return the current buffer size
+   */
+  float getSize();
+
+  /**
+   * Create {@link Flusher} implementation to flush the buffered rows to the underlying format
+   * implementation for faster processing.
+   *
+   * @param bdecVersion version of the BDEC file format to generate
+   * @return flusher
+   */
+  Flusher<T> createFlusher(Constants.BdecVersion bdecVersion);
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java
index 470b48557..9e5cd1b46 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java
@@ -9,12 +9,12 @@
 
 /** Builds a Streaming Ingest channel for a specific Streaming Ingest client */
 class SnowflakeStreamingIngestChannelFactory {
-  static SnowflakeStreamingIngestChannelBuilder builder(String name) {
-    return new SnowflakeStreamingIngestChannelBuilder(name);
+  static <T> SnowflakeStreamingIngestChannelBuilder<T> builder(String name) {
+    return new SnowflakeStreamingIngestChannelBuilder<>(name);
   }
 
   // Builder class to build a SnowflakeStreamingIngestChannel
-  static class SnowflakeStreamingIngestChannelBuilder {
+  static class SnowflakeStreamingIngestChannelBuilder<T> {
     private String name;
     private String dbName;
     private String schemaName;
@@ -22,7 +22,7 @@ static class SnowflakeStreamingIngestChannelBuilder {
     private String offsetToken;
     private Long channelSequencer;
     private Long rowSequencer;
-    private SnowflakeStreamingIngestClientInternal owningClient;
+    private SnowflakeStreamingIngestClientInternal<T> owningClient;
     private String encryptionKey;
     private Long encryptionKeyId;
     private OpenChannelRequest.OnErrorOption onErrorOption;
@@ -31,59 +31,59 @@ private SnowflakeStreamingIngestChannelBuilder(String name) {
       this.name = name;
     }
 
-    SnowflakeStreamingIngestChannelBuilder setDBName(String dbName) {
+    SnowflakeStreamingIngestChannelBuilder<T> setDBName(String dbName) {
       this.dbName = dbName;
       return this;
     }
 
-    SnowflakeStreamingIngestChannelBuilder setSchemaName(String schemaName) {
+    SnowflakeStreamingIngestChannelBuilder<T> setSchemaName(String schemaName) {
       this.schemaName = schemaName;
       return this;
     }
 
-    SnowflakeStreamingIngestChannelBuilder setTableName(String tableName) {
+    SnowflakeStreamingIngestChannelBuilder<T> setTableName(String tableName) {
       this.tableName = tableName;
       return this;
     }
 
-    SnowflakeStreamingIngestChannelBuilder setOffsetToken(String offsetToken) {
+    SnowflakeStreamingIngestChannelBuilder<T> setOffsetToken(String offsetToken) {
       this.offsetToken = offsetToken;
       return this;
     }
 
-    SnowflakeStreamingIngestChannelBuilder setChannelSequencer(Long sequencer) {
+    SnowflakeStreamingIngestChannelBuilder<T> setChannelSequencer(Long sequencer) {
       this.channelSequencer = sequencer;
       return this;
     }
 
-    SnowflakeStreamingIngestChannelBuilder setRowSequencer(Long sequencer) {
+    SnowflakeStreamingIngestChannelBuilder<T> setRowSequencer(Long sequencer) {
       this.rowSequencer = sequencer;
       return this;
     }
 
-    SnowflakeStreamingIngestChannelBuilder setEncryptionKey(String encryptionKey) {
+    SnowflakeStreamingIngestChannelBuilder<T> setEncryptionKey(String encryptionKey) {
       this.encryptionKey = encryptionKey;
       return this;
     }
 
-    SnowflakeStreamingIngestChannelBuilder setEncryptionKeyId(Long encryptionKeyId) {
+    SnowflakeStreamingIngestChannelBuilder<T> setEncryptionKeyId(Long encryptionKeyId) {
       this.encryptionKeyId = encryptionKeyId;
       return this;
     }
 
-    SnowflakeStreamingIngestChannelBuilder setOnErrorOption(
+    SnowflakeStreamingIngestChannelBuilder<T> setOnErrorOption(
         OpenChannelRequest.OnErrorOption onErrorOption) {
       this.onErrorOption = onErrorOption;
       return this;
     }
 
-    SnowflakeStreamingIngestChannelBuilder setOwningClient(
-        SnowflakeStreamingIngestClientInternal client) {
+    SnowflakeStreamingIngestChannelBuilder<T> setOwningClient(
+        SnowflakeStreamingIngestClientInternal<T> client) {
       this.owningClient = client;
       return this;
     }
 
-    SnowflakeStreamingIngestChannelInternal build() {
+    SnowflakeStreamingIngestChannelInternal<T> build() {
       Utils.assertStringNotNullOrEmpty("channel name", this.name);
       Utils.assertStringNotNullOrEmpty("table name", this.tableName);
       Utils.assertStringNotNullOrEmpty("schema name", this.schemaName);
@@ -94,7 +94,7 @@ SnowflakeStreamingIngestChannelInternal build() {
       Utils.assertStringNotNullOrEmpty("encryption key", this.encryptionKey);
       Utils.assertNotNull("encryption key_id", this.encryptionKeyId);
       Utils.assertNotNull("on_error option", this.onErrorOption);
-      return new SnowflakeStreamingIngestChannelInternal(
+      return new SnowflakeStreamingIngestChannelInternal<>(
           this.name,
           this.dbName,
           this.schemaName,
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 5eedb2855..b5ab6a811 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -22,9 +22,14 @@
 import net.snowflake.ingest.utils.SFException;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
 
-/** The first version of implementation for SnowflakeStreamingIngestChannel */
-class SnowflakeStreamingIngestChannelInternal implements SnowflakeStreamingIngestChannel {
+/**
+ * The first version of implementation for SnowflakeStreamingIngestChannel
+ *
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ */
+class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIngestChannel {
 
   private static final Logging logger = new Logging(SnowflakeStreamingIngestChannelInternal.class);
 
@@ -40,7 +45,7 @@ class SnowflakeStreamingIngestChannelInternal implements SnowflakeStreamingInges
   private final Long channelSequencer;
 
   // Reference to the row buffer
-  private final ArrowRowBuffer arrowBuffer;
+  private final RowBuffer<T> rowBuffer;
 
   // Indicates whether the channel is still valid
   private volatile boolean isValid;
@@ -49,7 +54,7 @@ class SnowflakeStreamingIngestChannelInternal implements SnowflakeStreamingInges
   private volatile boolean isClosed;
 
   // Reference to the client that owns this channel
-  private final SnowflakeStreamingIngestClientInternal owningClient;
+  private final SnowflakeStreamingIngestClientInternal<T> owningClient;
 
   // Memory allocator
   private final BufferAllocator allocator;
@@ -87,7 +92,7 @@ class SnowflakeStreamingIngestChannelInternal implements SnowflakeStreamingInges
       String offsetToken,
       Long channelSequencer,
       Long rowSequencer,
-      SnowflakeStreamingIngestClientInternal client,
+      SnowflakeStreamingIngestClientInternal<T> client,
       String encryptionKey,
       Long encryptionKeyId,
       OpenChannelRequest.OnErrorOption onErrorOption,
@@ -112,13 +117,22 @@ class SnowflakeStreamingIngestChannelInternal implements SnowflakeStreamingInges
                     String.format("%s_%s", name, channelSequencer),
                     0,
                     this.owningClient.getAllocator().getLimit());
-    this.arrowBuffer = new ArrowRowBuffer(this);
+    this.rowBuffer = createRowBuffer(client);
     this.encryptionKey = encryptionKey;
     this.encryptionKeyId = encryptionKeyId;
     this.onErrorOption = onErrorOption;
     logger.logInfo("Channel={} created for table={}", this.channelName, this.tableName);
   }
 
+  private RowBuffer<T> createRowBuffer(SnowflakeStreamingIngestClientInternal<T> client) {
+    // TODO: The circular dependency SnowflakeStreamingIngestChannelInternal <-> RowBuffer
+    // (SNOW-657667)
+    // can be probably reconsidered
+    //noinspection unchecked
+    return (RowBuffer<T>)
+        new ArrowRowBuffer((SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>) this);
+  }
+
   /**
    * Default Constructor
    *
@@ -139,7 +153,7 @@ class SnowflakeStreamingIngestChannelInternal implements SnowflakeStreamingInges
       String offsetToken,
       Long channelSequencer,
       Long rowSequencer,
-      SnowflakeStreamingIngestClientInternal client,
+      SnowflakeStreamingIngestClientInternal<T> client,
       String encryptionKey,
       Long encryptionKeyId,
       OpenChannelRequest.OnErrorOption onErrorOption) {
@@ -238,8 +252,8 @@ public String getFullyQualifiedTableName() {
    *
    * @return a ChannelData object
    */
-  ChannelData getData() {
-    return this.arrowBuffer.flush();
+  ChannelData<T> getData() {
+    return this.rowBuffer.flush();
   }
 
   /** @return a boolean to indicate whether the channel is valid or not */
@@ -251,7 +265,7 @@ public boolean isValid() {
   /** Mark the channel as invalid, and release resources */
   void invalidate() {
     this.isValid = false;
-    this.arrowBuffer.close("invalidate");
+    this.rowBuffer.close("invalidate");
     logger.logWarn(
         "Channel is invalidated, name={}, channel sequencer={}, row sequencer={}",
         getFullyQualifiedName(),
@@ -290,7 +304,7 @@ CompletableFuture<Void> flush(boolean closing) {
 
     // Simply return if there is no data in the channel, this might not work if we support public
     // flush API since there could a concurrent insert at the same time
-    if (this.arrowBuffer.getSize() == 0) {
+    if (this.rowBuffer.getSize() == 0) {
       return CompletableFuture.completedFuture(null);
     }
 
@@ -314,11 +328,11 @@ public CompletableFuture<Void> close() {
     return flush(true)
         .thenRunAsync(
             () -> {
-              List<SnowflakeStreamingIngestChannelInternal> uncommittedChannels =
+              List<SnowflakeStreamingIngestChannelInternal<?>> uncommittedChannels =
                   this.owningClient.verifyChannelsAreFullyCommitted(
                       Collections.singletonList(this));
 
-              this.arrowBuffer.close("close");
+              this.rowBuffer.close("close");
               this.owningClient.removeChannelIfSequencersMatch(this);
 
               // Throw an exception if the channel is invalid or has any uncommitted rows
@@ -349,7 +363,7 @@ BufferAllocator getAllocator() {
   // TODO: need to verify with the table schema when supporting sub-columns
   void setupSchema(List<ColumnMetadata> columns) {
     logger.logDebug("Setup schema for channel={}, schema={}", getFullyQualifiedName(), columns);
-    this.arrowBuffer.setupSchema(columns);
+    this.rowBuffer.setupSchema(columns);
   }
 
   /**
@@ -395,12 +409,12 @@ public InsertValidationResponse insertRows(
       throw new SFException(ErrorCode.CLOSED_CHANNEL, getFullyQualifiedName());
     }
 
-    InsertValidationResponse response = this.arrowBuffer.insertRows(rows, offsetToken);
+    InsertValidationResponse response = this.rowBuffer.insertRows(rows, offsetToken);
 
     // Start flush task if the chunk size reaches a certain size
     // TODO: Checking table/chunk level size reduces throughput a lot, we may want to check it only
     // if a large number of rows are inserted
-    if (this.arrowBuffer.getSize() >= MAX_CHUNK_SIZE_IN_BYTES) {
+    if (this.rowBuffer.getSize() >= MAX_CHUNK_SIZE_IN_BYTES) {
       this.owningClient.setNeedFlush();
     }
 
@@ -476,7 +490,7 @@ void throttleInsertIfNeeded(Runtime runtime) {
   private void checkValidation() {
     if (!isValid()) {
       this.owningClient.removeChannelIfSequencersMatch(this);
-      this.arrowBuffer.close("checkValidation");
+      this.rowBuffer.close("checkValidation");
       throw new SFException(ErrorCode.INVALID_CHANNEL, getFullyQualifiedName());
     }
   }
@@ -484,4 +498,9 @@ private void checkValidation() {
   OpenChannelRequest.OnErrorOption getOnErrorOption() {
     return this.onErrorOption;
   }
+
+  /** Returns underlying channel's row buffer implementation. */
+  RowBuffer<T> getRowBuffer() {
+    return rowBuffer;
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 086dd0f72..bb868a847 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -81,8 +81,10 @@
  * manages a few things:
  * <li>the channel cache, which contains all the channels that belong to this account
  * <li>the flush service, which schedules and coordinates the flush to Snowflake tables
+ *
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
-public class SnowflakeStreamingIngestClientInternal implements SnowflakeStreamingIngestClient {
+public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStreamingIngestClient {
 
   private static final Logging logger = new Logging(SnowflakeStreamingIngestClientInternal.class);
 
@@ -105,10 +107,10 @@ public class SnowflakeStreamingIngestClientInternal implements SnowflakeStreamin
   private final CloseableHttpClient httpClient;
 
   // Reference to the channel cache
-  private final ChannelCache channelCache;
+  private final ChannelCache<T> channelCache;
 
   // Reference to the flush service
-  private final FlushService flushService;
+  private final FlushService<T> flushService;
 
   // Memory allocator
   private final BufferAllocator allocator;
@@ -164,7 +166,7 @@ public class SnowflakeStreamingIngestClientInternal implements SnowflakeStreamin
     this.name = name;
     this.isTestMode = isTestMode;
     this.httpClient = httpClient == null ? HttpUtil.getHttpClient() : httpClient;
-    this.channelCache = new ChannelCache();
+    this.channelCache = new ChannelCache<>();
     this.allocator = new RootAllocator();
     this.isClosed = false;
     this.requestBuilder = requestBuilder;
@@ -190,7 +192,7 @@ public class SnowflakeStreamingIngestClientInternal implements SnowflakeStreamin
       this.setupMetricsForClient();
     }
 
-    this.flushService = new FlushService(this, this.channelCache, this.isTestMode);
+    this.flushService = new FlushService<>(this, this.channelCache, this.isTestMode);
 
     logger.logInfo(
         "Client created, name={}, account={}. isTestMode={}, parameters={}",
@@ -257,7 +259,7 @@ public boolean isClosed() {
    * @return a SnowflakeStreamingIngestChannel object
    */
   @Override
-  public SnowflakeStreamingIngestChannelInternal openChannel(OpenChannelRequest request) {
+  public SnowflakeStreamingIngestChannelInternal<?> openChannel(OpenChannelRequest request) {
     if (isClosed) {
       throw new SFException(ErrorCode.CLOSED_CLIENT);
     }
@@ -307,8 +309,8 @@ public SnowflakeStreamingIngestChannelInternal openChannel(OpenChannelRequest re
           getName());
 
       // Channel is now registered, add it to the in-memory channel pool
-      SnowflakeStreamingIngestChannelInternal channel =
-          SnowflakeStreamingIngestChannelFactory.builder(response.getChannelName())
+      SnowflakeStreamingIngestChannelInternal<T> channel =
+          SnowflakeStreamingIngestChannelFactory.<T>builder(response.getChannelName())
               .setDBName(response.getDBName())
               .setSchemaName(response.getSchemaName())
               .setTableName(response.getTableName())
@@ -339,7 +341,8 @@ public SnowflakeStreamingIngestChannelInternal openChannel(OpenChannelRequest re
    * @param channels a list of channels that we want to get the status on
    * @return a ChannelsStatusResponse object
    */
-  ChannelsStatusResponse getChannelsStatus(List<SnowflakeStreamingIngestChannelInternal> channels) {
+  ChannelsStatusResponse getChannelsStatus(
+      List<SnowflakeStreamingIngestChannelInternal<?>> channels) {
     try {
       ChannelsStatusRequest request = new ChannelsStatusRequest();
       List<ChannelsStatusRequest.ChannelStatusRequestDTO> requestDTOs =
@@ -368,7 +371,7 @@ ChannelsStatusResponse getChannelsStatus(List<SnowflakeStreamingIngestChannelInt
       }
 
       for (int idx = 0; idx < channels.size(); idx++) {
-        SnowflakeStreamingIngestChannelInternal channel = channels.get(idx);
+        SnowflakeStreamingIngestChannelInternal<?> channel = channels.get(idx);
         ChannelsStatusResponse.ChannelStatusResponseDTO channelStatus =
             response.getChannels().get(idx);
         if (channelStatus.getStatusCode() != RESPONSE_SUCCESS) {
@@ -631,7 +634,7 @@ BufferAllocator getAllocator() {
   }
 
   /** Remove the channel in the channel cache if the channel sequencer matches */
-  void removeChannelIfSequencersMatch(SnowflakeStreamingIngestChannelInternal channel) {
+  void removeChannelIfSequencersMatch(SnowflakeStreamingIngestChannelInternal<T> channel) {
     this.channelCache.removeChannelIfSequencersMatch(channel);
   }
 
@@ -651,12 +654,12 @@ RequestBuilder getRequestBuilder() {
   }
 
   /** Get the channel cache */
-  ChannelCache getChannelCache() {
+  ChannelCache<T> getChannelCache() {
     return this.channelCache;
   }
 
   /** Get the flush service */
-  FlushService getFlushService() {
+  FlushService<T> getFlushService() {
     return this.flushService;
   }
 
@@ -666,8 +669,8 @@ FlushService getFlushService() {
    * @param channels a list of channels we want to check against
    * @return a list of channels that has uncommitted rows
    */
-  List<SnowflakeStreamingIngestChannelInternal> verifyChannelsAreFullyCommitted(
-      List<SnowflakeStreamingIngestChannelInternal> channels) {
+  List<SnowflakeStreamingIngestChannelInternal<?>> verifyChannelsAreFullyCommitted(
+      List<SnowflakeStreamingIngestChannelInternal<?>> channels) {
     if (channels.isEmpty()) {
       return channels;
     }
@@ -676,16 +679,16 @@ List<SnowflakeStreamingIngestChannelInternal> verifyChannelsAreFullyCommitted(
     int retry = 0;
     boolean isTimeout = true;
     List<ChannelsStatusResponse.ChannelStatusResponseDTO> oldChannelsStatus = new ArrayList<>();
-    List<SnowflakeStreamingIngestChannelInternal> channelsWithError = new ArrayList<>();
+    List<SnowflakeStreamingIngestChannelInternal<?>> channelsWithError = new ArrayList<>();
     do {
       List<ChannelsStatusResponse.ChannelStatusResponseDTO> channelsStatus =
           getChannelsStatus(channels).getChannels();
-      List<SnowflakeStreamingIngestChannelInternal> tempChannels = new ArrayList<>();
+      List<SnowflakeStreamingIngestChannelInternal<?>> tempChannels = new ArrayList<>();
       List<ChannelsStatusResponse.ChannelStatusResponseDTO> tempChannelsStatus = new ArrayList<>();
 
       for (int idx = 0; idx < channelsStatus.size(); idx++) {
         ChannelsStatusResponse.ChannelStatusResponseDTO channelStatus = channelsStatus.get(idx);
-        SnowflakeStreamingIngestChannelInternal channel = channels.get(idx);
+        SnowflakeStreamingIngestChannelInternal<?> channel = channels.get(idx);
         logger.logInfo(
             "Get channel status name={}, status={}, clientSequencer={}, rowSequencer={},"
                 + " offsetToken={}, persistedRowSequencer={}, persistedOffsetToken={}",
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index 8aa09868b..4d255e202 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -79,7 +79,7 @@ public enum ArrowBatchWriteMode {
   }
 
   /** The write mode to generate Arrow BDEC file. */
-  public enum BdecVerion {
+  public enum BdecVersion {
     /** Uses Arrow to generate BDEC chunks with {@link ArrowBatchWriteMode#STREAM}. */
     ONE(1),
 
@@ -88,7 +88,7 @@ public enum BdecVerion {
 
     private final byte version;
 
-    BdecVerion(int version) {
+    BdecVersion(int version) {
       if (version > Byte.MAX_VALUE || version < Byte.MIN_VALUE) {
         throw new IllegalArgumentException("Version does not fit into the byte data type");
       }
@@ -99,12 +99,12 @@ public byte toByte() {
       return version;
     }
 
-    public static BdecVerion fromInt(int val) {
+    public static BdecVersion fromInt(int val) {
       if (val > Byte.MAX_VALUE || val < Byte.MIN_VALUE) {
         throw new IllegalArgumentException("Version does not fit into the byte data type");
       }
       byte version = (byte) val;
-      for (BdecVerion eversion : BdecVerion.values()) {
+      for (BdecVersion eversion : BdecVersion.values()) {
         if (eversion.version == version) {
           return eversion;
         }
@@ -112,7 +112,7 @@ public static BdecVerion fromInt(int val) {
       throw new IllegalArgumentException(
           String.format(
               "Unsupported BLOB_FORMAT_VERSION = '%d', allowed values are %s",
-              version, Arrays.asList(BdecVerion.values())));
+              version, Arrays.asList(BdecVersion.values())));
     }
   }
 
diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
index 1882fec04..3289df694 100644
--- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
+++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -26,7 +26,7 @@ public class ParameterProvider {
   public static final long INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_DEFAULT = 10;
   public static final boolean SNOWPIPE_STREAMING_METRICS_DEFAULT = false;
 
-  public static final Constants.BdecVerion BLOB_FORMAT_VERSION_DEFAULT = Constants.BdecVerion.ONE;
+  public static final Constants.BdecVersion BLOB_FORMAT_VERSION_DEFAULT = Constants.BdecVersion.ONE;
 
   /** Map of parameter name to parameter value. This will be set by client/configure API Call. */
   private final Map<String, Object> parameterMap = new HashMap<>();
@@ -138,10 +138,10 @@ public boolean hasEnabledSnowpipeStreamingMetrics() {
   }
 
   /** @return Blob format version: 1 (arrow stream write mode), 2 (arrow file write mode) etc */
-  public Constants.BdecVerion getBlobFormatVersion() {
+  public Constants.BdecVersion getBlobFormatVersion() {
     Object val = this.parameterMap.getOrDefault(BLOB_FORMAT_VERSION, BLOB_FORMAT_VERSION_DEFAULT);
-    if (val instanceof Constants.BdecVerion) {
-      return (Constants.BdecVerion) val;
+    if (val instanceof Constants.BdecVersion) {
+      return (Constants.BdecVersion) val;
     }
     if (val instanceof String) {
       try {
@@ -151,7 +151,7 @@ public Constants.BdecVerion getBlobFormatVersion() {
             String.format("Failed to parse BLOB_FORMAT_VERSION = '%s'", val), t);
       }
     }
-    return Constants.BdecVerion.fromInt((int) val);
+    return Constants.BdecVersion.fromInt((int) val);
   }
 
   @Override
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java
index 8f98bf76d..ba909d62d 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java
@@ -4,16 +4,17 @@
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
+import org.apache.arrow.vector.VectorSchemaRoot;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
 public class ChannelCacheTest {
-  ChannelCache cache;
-  SnowflakeStreamingIngestClientInternal client;
-  SnowflakeStreamingIngestChannelInternal channel1;
-  SnowflakeStreamingIngestChannelInternal channel2;
-  SnowflakeStreamingIngestChannelInternal channel3;
+  ChannelCache<VectorSchemaRoot> cache;
+  SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client;
+  SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel1;
+  SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel2;
+  SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel3;
   String dbName = "db";
   String schemaName = "schema";
   String table1Name = "table1";
@@ -21,10 +22,10 @@ public class ChannelCacheTest {
 
   @Before
   public void setup() {
-    cache = new ChannelCache();
-    client = new SnowflakeStreamingIngestClientInternal("client");
+    cache = new ChannelCache<>();
+    client = new SnowflakeStreamingIngestClientInternal<>("client");
     channel1 =
-        new SnowflakeStreamingIngestChannelInternal(
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel1",
             dbName,
             schemaName,
@@ -38,7 +39,7 @@ public void setup() {
             OpenChannelRequest.OnErrorOption.CONTINUE,
             true);
     channel2 =
-        new SnowflakeStreamingIngestChannelInternal(
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel2",
             dbName,
             schemaName,
@@ -52,7 +53,7 @@ public void setup() {
             OpenChannelRequest.OnErrorOption.CONTINUE,
             true);
     channel3 =
-        new SnowflakeStreamingIngestChannelInternal(
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel3",
             dbName,
             schemaName,
@@ -75,10 +76,10 @@ public void testAddChannel() {
     String channelName = "channel";
     String tableName = "table";
 
-    ChannelCache cache = new ChannelCache();
+    ChannelCache<VectorSchemaRoot> cache = new ChannelCache<>();
     Assert.assertEquals(0, cache.getSize());
-    SnowflakeStreamingIngestChannelInternal channel =
-        new SnowflakeStreamingIngestChannelInternal(
+    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
             channelName,
             dbName,
             schemaName,
@@ -95,8 +96,8 @@ public void testAddChannel() {
     Assert.assertEquals(1, cache.getSize());
     Assert.assertTrue(channel == cache.iterator().next().getValue().get(channelName));
 
-    SnowflakeStreamingIngestChannelInternal channelDup =
-        new SnowflakeStreamingIngestChannelInternal(
+    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channelDup =
+        new SnowflakeStreamingIngestChannelInternal<>(
             channelName,
             dbName,
             schemaName,
@@ -114,7 +115,7 @@ public void testAddChannel() {
     Assert.assertTrue(!channel.isValid());
     Assert.assertTrue(channelDup.isValid());
     Assert.assertEquals(1, cache.getSize());
-    ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal> channels =
+    ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>> channels =
         cache.iterator().next().getValue();
     Assert.assertEquals(1, channels.size());
     Assert.assertTrue(channelDup == channels.get(channelName));
@@ -124,11 +125,19 @@ public void testAddChannel() {
   @Test
   public void testIterator() {
     Assert.assertEquals(2, cache.getSize());
-    Iterator<Map.Entry<String, ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal>>>
+    Iterator<
+            Map.Entry<
+                String,
+                ConcurrentHashMap<
+                    String, SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>>>>
         iter = cache.iterator();
-    Map.Entry<String, ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal>>
+    Map.Entry<
+            String,
+            ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>>>
         firstTable = iter.next();
-    Map.Entry<String, ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal>>
+    Map.Entry<
+            String,
+            ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>>>
         secondTable = iter.next();
     Assert.assertFalse(iter.hasNext());
     if (firstTable.getKey().equals(channel1.getFullyQualifiedTableName())) {
@@ -147,10 +156,14 @@ public void testIterator() {
   @Test
   public void testCloseAllChannels() {
     cache.closeAllChannels();
-    Iterator<Map.Entry<String, ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal>>>
+    Iterator<
+            Map.Entry<
+                String,
+                ConcurrentHashMap<
+                    String, SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>>>>
         iter = cache.iterator();
     while (iter.hasNext()) {
-      for (SnowflakeStreamingIngestChannelInternal channel : iter.next().getValue().values()) {
+      for (SnowflakeStreamingIngestChannelInternal<?> channel : iter.next().getValue().values()) {
         Assert.assertTrue(channel.isClosed());
       }
     }
@@ -166,8 +179,8 @@ public void testRemoveChannel() {
     cache.removeChannelIfSequencersMatch(channel2);
     Assert.assertEquals(1, cache.getSize());
 
-    SnowflakeStreamingIngestChannelInternal channel3Dup =
-        new SnowflakeStreamingIngestChannelInternal(
+    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel3Dup =
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel3",
             dbName,
             schemaName,
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 7e08ca6cf..58746b8f7 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -53,12 +53,12 @@
 import org.mockito.Mockito;
 
 public class FlushServiceTest {
-  private SnowflakeStreamingIngestClientInternal client;
-  private ChannelCache channelCache;
+  private SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client;
+  private ChannelCache<VectorSchemaRoot> channelCache;
   private SnowflakeConnectionV1 conn;
-  private SnowflakeStreamingIngestChannelInternal channel1;
-  private SnowflakeStreamingIngestChannelInternal channel2;
-  private SnowflakeStreamingIngestChannelInternal channel3;
+  private SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel1;
+  private SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel2;
+  private SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel3;
   private StreamingIngestStage stage;
 
   private final BufferAllocator allocator = new RootAllocator();
@@ -72,11 +72,11 @@ public void setup() {
     Mockito.when(client.getParameterProvider()).thenReturn(parameterProvider);
     stage = Mockito.mock(StreamingIngestStage.class);
     Mockito.when(stage.getClientPrefix()).thenReturn("client_prefix");
-    channelCache = new ChannelCache();
+    channelCache = new ChannelCache<>();
     Mockito.when(client.getChannelCache()).thenReturn(channelCache);
     conn = Mockito.mock(SnowflakeConnectionV1.class);
     channel1 =
-        new SnowflakeStreamingIngestChannelInternal(
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel1",
             "db1",
             "schema1",
@@ -91,7 +91,7 @@ public void setup() {
             true);
 
     channel2 =
-        new SnowflakeStreamingIngestChannelInternal(
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel2",
             "db1",
             "schema1",
@@ -106,7 +106,7 @@ public void setup() {
             true);
 
     channel3 =
-        new SnowflakeStreamingIngestChannelInternal(
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel3",
             "db2",
             "schema1",
@@ -126,7 +126,8 @@ public void setup() {
 
   @Test
   public void testGetFilePath() {
-    FlushService flushService = new FlushService(client, channelCache, stage, false);
+    FlushService<VectorSchemaRoot> flushService =
+        new FlushService<>(client, channelCache, stage, false);
     Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
     String clientPrefix = "honk";
     String outputString = flushService.getFilePath(calendar, clientPrefix);
@@ -159,7 +160,8 @@ public void testGetFilePath() {
 
   @Test
   public void testFlush() throws Exception {
-    FlushService flushService = Mockito.spy(new FlushService(client, channelCache, stage, false));
+    FlushService<VectorSchemaRoot> flushService =
+        Mockito.spy(new FlushService<>(client, channelCache, stage, false));
 
     // Nothing to flush
     flushService.flush(false).get();
@@ -185,14 +187,15 @@ public void testFlush() throws Exception {
 
   @Test
   public void testBuildAndUpload() throws Exception {
-    FlushService flushService = Mockito.spy(new FlushService(client, channelCache, stage, true));
+    FlushService<VectorSchemaRoot> flushService =
+        Mockito.spy(new FlushService<>(client, channelCache, stage, true));
 
-    List<List<ChannelData>> blobData = new ArrayList<>();
-    List<ChannelData> chunkData = new ArrayList<>();
+    List<List<ChannelData<VectorSchemaRoot>>> blobData = new ArrayList<>();
+    List<ChannelData<VectorSchemaRoot>> chunkData = new ArrayList<>();
 
     // Construct fields
-    ChannelData channel1Data = new ChannelData();
-    ChannelData channel2Data = new ChannelData();
+    ChannelData<VectorSchemaRoot> channel1Data = new ChannelData<>();
+    ChannelData<VectorSchemaRoot> channel2Data = new ChannelData<>();
 
     FieldVector vector1 = new VarCharVector("vector1", allocator);
     FieldVector vector2 = new IntVector("vector2", allocator);
@@ -253,16 +256,19 @@ public void testBuildAndUpload() throws Exception {
     channel1Data.setOffsetToken("offset1");
     channel1Data.setBufferSize(100);
     channel1Data.setChannel(channel1);
+    channel1Data.setRowCount(2);
 
     channel2Data.setRowSequencer(10L);
     channel2Data.setOffsetToken("offset2");
     channel2Data.setBufferSize(100);
     channel2Data.setChannel(channel2);
+    channel2Data.setRowCount(1);
 
     BlobMetadata blobMetadata = flushService.buildAndUpload("file_name", blobData);
 
     EpInfo expectedChunkEpInfo =
-        ArrowRowBuffer.buildEpInfoFromStats(3, ChannelData.getCombinedColumnStatsMap(eps1, eps2));
+        AbstractRowBuffer.buildEpInfoFromStats(
+            3, ChannelData.getCombinedColumnStatsMap(eps1, eps2));
 
     ChannelMetadata expectedChannel1Metadata =
         ChannelMetadata.builder()
@@ -344,14 +350,15 @@ public void testBuildAndUpload() throws Exception {
   @Test
   public void testBuildErrors() throws Exception {
     // build should error if we try to group channels for different tables together
-    FlushService flushService = Mockito.spy(new FlushService(client, channelCache, stage, true));
+    FlushService<VectorSchemaRoot> flushService =
+        Mockito.spy(new FlushService<>(client, channelCache, stage, true));
 
-    List<List<ChannelData>> channelData = new ArrayList<>();
-    List<ChannelData> channelData1 = new ArrayList<>();
+    List<List<ChannelData<VectorSchemaRoot>>> channelData = new ArrayList<>();
+    List<ChannelData<VectorSchemaRoot>> channelData1 = new ArrayList<>();
 
     // Construct fields
-    ChannelData data1 = new ChannelData();
-    ChannelData data2 = new ChannelData();
+    ChannelData<VectorSchemaRoot> data1 = new ChannelData<>();
+    ChannelData<VectorSchemaRoot> data2 = new ChannelData<>();
 
     FieldVector vector1 = new VarCharVector("vector1", allocator);
     FieldVector vector3 = new IntVector("vector3", allocator);
@@ -402,14 +409,14 @@ public void testBuildErrors() throws Exception {
   @Test
   public void testInvalidateChannels() throws Exception {
     // Create a new Client in order to not interfere with other tests
-    SnowflakeStreamingIngestClientInternal client =
+    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
         Mockito.mock(SnowflakeStreamingIngestClientInternal.class);
     ParameterProvider parameterProvider = new ParameterProvider();
-    ChannelCache channelCache = new ChannelCache();
+    ChannelCache<VectorSchemaRoot> channelCache = new ChannelCache<>();
     Mockito.when(client.getChannelCache()).thenReturn(channelCache);
     Mockito.when(client.getParameterProvider()).thenReturn(parameterProvider);
-    SnowflakeStreamingIngestChannelInternal channel1 =
-        new SnowflakeStreamingIngestChannelInternal(
+    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel1 =
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel1",
             "db1",
             "schema1",
@@ -423,8 +430,8 @@ public void testInvalidateChannels() throws Exception {
             OpenChannelRequest.OnErrorOption.CONTINUE,
             true);
 
-    SnowflakeStreamingIngestChannelInternal channel2 =
-        new SnowflakeStreamingIngestChannelInternal(
+    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel2 =
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel2",
             "db1",
             "schema1",
@@ -441,12 +448,12 @@ public void testInvalidateChannels() throws Exception {
     channelCache.addChannel(channel1);
     channelCache.addChannel(channel2);
 
-    List<List<ChannelData>> blobData = new ArrayList<>();
-    List<ChannelData> innerData = new ArrayList<>();
+    List<List<ChannelData<VectorSchemaRoot>>> blobData = new ArrayList<>();
+    List<ChannelData<VectorSchemaRoot>> innerData = new ArrayList<>();
     blobData.add(innerData);
 
-    ChannelData channel1Data = new ChannelData();
-    ChannelData channel2Data = new ChannelData();
+    ChannelData<VectorSchemaRoot> channel1Data = new ChannelData<>();
+    ChannelData<VectorSchemaRoot> channel2Data = new ChannelData<>();
     channel1Data.setChannel(channel1);
     channel2Data.setChannel(channel1);
 
@@ -469,7 +476,8 @@ public void testInvalidateChannels() throws Exception {
     innerData.add(channel1Data);
     innerData.add(channel2Data);
 
-    FlushService flushService = new FlushService(client, channelCache, stage, false);
+    FlushService<VectorSchemaRoot> flushService =
+        new FlushService<>(client, channelCache, stage, false);
     flushService.invalidateAllChannelsInBlob(blobData);
 
     Assert.assertFalse(channel1.isValid());
@@ -495,7 +503,7 @@ public void testBlobBuilder() throws Exception {
 
     stats1.addIntValue(new BigInteger("10"));
     stats1.addIntValue(new BigInteger("15"));
-    EpInfo epInfo = ArrowRowBuffer.buildEpInfoFromStats(2, eps1);
+    EpInfo epInfo = AbstractRowBuffer.buildEpInfoFromStats(2, eps1);
 
     ChannelMetadata channelMetadata =
         ChannelMetadata.builder()
@@ -516,7 +524,7 @@ public void testBlobBuilder() throws Exception {
 
     chunksMetadataList.add(chunkMetadata);
 
-    final Constants.BdecVerion bdecVersion = ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT;
+    final Constants.BdecVersion bdecVersion = ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT;
     byte[] blob =
         BlobBuilder.build(chunksMetadataList, chunksDataList, checksum, dataSize, bdecVersion);
 
@@ -570,7 +578,8 @@ public void testBlobBuilder() throws Exception {
 
   @Test
   public void testShutDown() throws Exception {
-    FlushService flushService = new FlushService(client, channelCache, stage, false);
+    FlushService<VectorSchemaRoot> flushService =
+        new FlushService<>(client, channelCache, stage, false);
 
     Assert.assertFalse(flushService.buildUploadWorkers.isShutdown());
     Assert.assertFalse(flushService.registerWorker.isShutdown());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
index d8cda59be..53f96808a 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
@@ -10,6 +10,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import net.snowflake.ingest.utils.Pair;
+import org.apache.arrow.vector.VectorSchemaRoot;
 import org.junit.Assert;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -18,15 +19,15 @@ public class RegisterServiceTest {
 
   @Test
   public void testRegisterService() throws Exception {
-    RegisterService rs = new RegisterService(null, true);
+    RegisterService<VectorSchemaRoot> rs = new RegisterService<>(null, true);
 
-    Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>> blobFuture =
+    Pair<FlushService.BlobData<VectorSchemaRoot>, CompletableFuture<BlobMetadata>> blobFuture =
         new Pair<>(
-            new FlushService.BlobData("test", null),
+            new FlushService.BlobData<>("test", null),
             CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null)));
     rs.addBlobs(Collections.singletonList(blobFuture));
     Assert.assertEquals(1, rs.getBlobsList().size());
-    List<FlushService.BlobData> errorBlobs = rs.registerBlobs(null);
+    List<FlushService.BlobData<VectorSchemaRoot>> errorBlobs = rs.registerBlobs(null);
     Assert.assertEquals(0, rs.getBlobsList().size());
     Assert.assertEquals(0, errorBlobs.size());
   }
@@ -42,22 +43,22 @@ public void testRegisterService() throws Exception {
    */
   @Test
   public void testRegisterServiceTimeoutException() throws Exception {
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal("client");
-    RegisterService rs = new RegisterService(client, true);
+    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
+        new SnowflakeStreamingIngestClientInternal<>("client");
+    RegisterService<VectorSchemaRoot> rs = new RegisterService<>(client, true);
 
-    Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>> blobFuture1 =
+    Pair<FlushService.BlobData<VectorSchemaRoot>, CompletableFuture<BlobMetadata>> blobFuture1 =
         new Pair<>(
-            new FlushService.BlobData("success", new ArrayList<>()),
+            new FlushService.BlobData<>("success", new ArrayList<>()),
             CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null)));
     CompletableFuture future = new CompletableFuture();
     future.completeExceptionally(new TimeoutException());
-    Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>> blobFuture2 =
-        new Pair<>(new FlushService.BlobData("fail", new ArrayList<>()), future);
+    Pair<FlushService.BlobData<VectorSchemaRoot>, CompletableFuture<BlobMetadata>> blobFuture2 =
+        new Pair<>(new FlushService.BlobData<VectorSchemaRoot>("fail", new ArrayList<>()), future);
     rs.addBlobs(Arrays.asList(blobFuture1, blobFuture2));
     Assert.assertEquals(2, rs.getBlobsList().size());
     try {
-      List<FlushService.BlobData> errorBlobs = rs.registerBlobs(null);
+      List<FlushService.BlobData<VectorSchemaRoot>> errorBlobs = rs.registerBlobs(null);
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
       Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
@@ -70,13 +71,13 @@ public void testRegisterServiceTimeoutException() throws Exception {
   @Ignore
   @Test
   public void testRegisterServiceTimeoutException_testRetries() throws Exception {
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal("client");
-    RegisterService rs = new RegisterService(client, true);
+    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
+        new SnowflakeStreamingIngestClientInternal<>("client");
+    RegisterService<VectorSchemaRoot> rs = new RegisterService<>(client, true);
 
-    Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>> blobFuture1 =
+    Pair<FlushService.BlobData<VectorSchemaRoot>, CompletableFuture<BlobMetadata>> blobFuture1 =
         new Pair<>(
-            new FlushService.BlobData("success", new ArrayList<>()),
+            new FlushService.BlobData<>("success", new ArrayList<>()),
             CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null)));
     CompletableFuture future = new CompletableFuture();
     future.thenRunAsync(
@@ -88,12 +89,12 @@ public void testRegisterServiceTimeoutException_testRetries() throws Exception {
           }
           return;
         });
-    Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>> blobFuture2 =
-        new Pair<>(new FlushService.BlobData("fail", new ArrayList<>()), future);
+    Pair<FlushService.BlobData<VectorSchemaRoot>, CompletableFuture<BlobMetadata>> blobFuture2 =
+        new Pair<>(new FlushService.BlobData<VectorSchemaRoot>("fail", new ArrayList<>()), future);
     rs.addBlobs(Arrays.asList(blobFuture1, blobFuture2));
     Assert.assertEquals(2, rs.getBlobsList().size());
     try {
-      List<FlushService.BlobData> errorBlobs = rs.registerBlobs(null);
+      List<FlushService.BlobData<VectorSchemaRoot>> errorBlobs = rs.registerBlobs(null);
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
       Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
@@ -104,18 +105,18 @@ public void testRegisterServiceTimeoutException_testRetries() throws Exception {
 
   @Test
   public void testRegisterServiceNonTimeoutException() {
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal("client");
-    RegisterService rs = new RegisterService(client, true);
+    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
+        new SnowflakeStreamingIngestClientInternal<>("client");
+    RegisterService<VectorSchemaRoot> rs = new RegisterService<>(client, true);
 
     CompletableFuture future = new CompletableFuture();
     future.completeExceptionally(new IndexOutOfBoundsException());
-    Pair<FlushService.BlobData, CompletableFuture<BlobMetadata>> blobFuture =
-        new Pair<>(new FlushService.BlobData("fail", new ArrayList<>()), future);
+    Pair<FlushService.BlobData<VectorSchemaRoot>, CompletableFuture<BlobMetadata>> blobFuture =
+        new Pair<>(new FlushService.BlobData<VectorSchemaRoot>("fail", new ArrayList<>()), future);
     rs.addBlobs(Collections.singletonList(blobFuture));
     Assert.assertEquals(1, rs.getBlobsList().size());
     try {
-      List<FlushService.BlobData> errorBlobs = rs.registerBlobs(null);
+      List<FlushService.BlobData<VectorSchemaRoot>> errorBlobs = rs.registerBlobs(null);
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
       Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 3015099e0..fcf896e40 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -16,6 +16,7 @@
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.SFException;
+import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
@@ -28,18 +29,18 @@ public class RowBufferTest {
   private static final Logging logger = new Logging(RegisterService.class);
 
   private ArrowRowBuffer rowBufferOnErrorContinue;
-  private SnowflakeStreamingIngestChannelInternal channelOnErrorContinue;
+  private SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channelOnErrorContinue;
 
   private ArrowRowBuffer rowBufferOnErrorAbort;
-  private SnowflakeStreamingIngestChannelInternal channelOnErrorAbort;
+  private SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channelOnErrorAbort;
 
   @Before
   public void setupRowBuffer() {
     // Create row buffer
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal("client");
+    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
+        new SnowflakeStreamingIngestClientInternal<>("client");
     this.channelOnErrorContinue =
-        new SnowflakeStreamingIngestChannelInternal(
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel",
             "db",
             "schema",
@@ -55,7 +56,7 @@ public void setupRowBuffer() {
     this.rowBufferOnErrorContinue = new ArrowRowBuffer(this.channelOnErrorContinue);
 
     this.channelOnErrorAbort =
-        new SnowflakeStreamingIngestChannelInternal(
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel",
             "db",
             "schema",
@@ -766,7 +767,7 @@ private void testFlushHelper(ArrowRowBuffer rowBuffer) {
     Assert.assertFalse(response.hasErrors());
     float bufferSize = rowBuffer.getSize();
 
-    ChannelData data = rowBuffer.flush();
+    ChannelData<VectorSchemaRoot> data = rowBuffer.flush();
     Assert.assertEquals(2, data.getRowCount());
     Assert.assertEquals((Long) 1L, data.getRowSequencer());
     Assert.assertEquals(7, data.getVectors().getFieldVectors().size());
@@ -780,7 +781,8 @@ public void testDoubleQuotesColumnName() {
     testDoubleQuotesColumnNameHelper(this.channelOnErrorContinue);
   }
 
-  private void testDoubleQuotesColumnNameHelper(SnowflakeStreamingIngestChannelInternal channel) {
+  private void testDoubleQuotesColumnNameHelper(
+      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
     ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
 
     ColumnMetadata colDoubleQuotes = new ColumnMetadata();
@@ -818,7 +820,7 @@ public void testBuildEpInfoFromStats() {
     colStats.put("intColumn", stats1);
     colStats.put("strColumn", stats2);
 
-    EpInfo result = ArrowRowBuffer.buildEpInfoFromStats(2, colStats);
+    EpInfo result = AbstractRowBuffer.buildEpInfoFromStats(2, colStats);
     Map<String, FileColumnProperties> columnResults = result.getColumnEps();
     Assert.assertEquals(2, columnResults.keySet().size());
 
@@ -847,7 +849,7 @@ public void testBuildEpInfoFromNullColumnStats() {
     colStats.put(intColName, stats);
     colStats.put(realColName, stats);
 
-    EpInfo result = ArrowRowBuffer.buildEpInfoFromStats(2, colStats);
+    EpInfo result = AbstractRowBuffer.buildEpInfoFromStats(2, colStats);
     Map<String, FileColumnProperties> columnResults = result.getColumnEps();
     Assert.assertEquals(2, columnResults.keySet().size());
 
@@ -905,7 +907,8 @@ public void testArrowE2ETimestampLTZ() {
     testArrowE2ETimestampLTZHelper(this.channelOnErrorAbort);
   }
 
-  private void testArrowE2ETimestampLTZHelper(SnowflakeStreamingIngestChannelInternal channel) {
+  private void testArrowE2ETimestampLTZHelper(
+      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
     ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
 
     ColumnMetadata colTimestampLtzSB8 = new ColumnMetadata();
@@ -973,7 +976,8 @@ public void testArrowE2ETimestampTZ() {
     testArrowE2ETimestampTZHelper(this.channelOnErrorAbort);
   }
 
-  private void testArrowE2ETimestampTZHelper(SnowflakeStreamingIngestChannelInternal channel) {
+  private void testArrowE2ETimestampTZHelper(
+      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
     ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
 
     ColumnMetadata colTimestampTzSB8 = new ColumnMetadata();
@@ -1094,7 +1098,8 @@ public void testArrowE2ETimestampErrors() {
     testArrowE2ETimestampErrorsHelper(this.channelOnErrorContinue);
   }
 
-  private void testArrowE2ETimestampErrorsHelper(SnowflakeStreamingIngestChannelInternal channel) {
+  private void testArrowE2ETimestampErrorsHelper(
+      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
     ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
 
     ColumnMetadata colTimestampLtzSB16 = new ColumnMetadata();
@@ -1154,7 +1159,7 @@ private void testStatsE2EHelper(ArrowRowBuffer rowBuffer) {
 
     InsertValidationResponse response = rowBuffer.insertRows(Arrays.asList(row1, row2), null);
     Assert.assertFalse(response.hasErrors());
-    ChannelData result = rowBuffer.flush();
+    ChannelData<VectorSchemaRoot> result = rowBuffer.flush();
     Map<String, RowBufferStats> columnEpStats = result.getColumnEps();
 
     Assert.assertEquals(
@@ -1196,7 +1201,7 @@ private void testStatsE2EHelper(ArrowRowBuffer rowBuffer) {
     Assert.assertEquals(-1, columnEpStats.get("COLCHAR").getDistinctValues());
 
     // Confirm we reset
-    ChannelData resetResults = rowBuffer.flush();
+    ChannelData<VectorSchemaRoot> resetResults = rowBuffer.flush();
     Assert.assertNull(resetResults);
   }
 
@@ -1206,7 +1211,8 @@ public void testStatsE2ETimestamp() {
     testStatsE2ETimestampHelper(this.channelOnErrorContinue);
   }
 
-  private void testStatsE2ETimestampHelper(SnowflakeStreamingIngestChannelInternal channel) {
+  private void testStatsE2ETimestampHelper(
+      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
     ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
 
     ColumnMetadata colTimestampLtzSB8 = new ColumnMetadata();
@@ -1251,7 +1257,7 @@ private void testStatsE2ETimestampHelper(SnowflakeStreamingIngestChannelInternal
     InsertValidationResponse response =
         innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null);
     Assert.assertFalse(response.hasErrors());
-    ChannelData result = innerBuffer.flush();
+    ChannelData<VectorSchemaRoot> result = innerBuffer.flush();
     Assert.assertEquals(3, result.getRowCount());
 
     Assert.assertEquals(
@@ -1287,7 +1293,8 @@ public void testE2EDate() {
     testE2EDateHelper(this.channelOnErrorAbort);
   }
 
-  private void testE2EDateHelper(SnowflakeStreamingIngestChannelInternal channel) {
+  private void testE2EDateHelper(
+      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
     ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
 
     ColumnMetadata colDate = new ColumnMetadata();
@@ -1318,7 +1325,7 @@ private void testE2EDateHelper(SnowflakeStreamingIngestChannelInternal channel)
     Assert.assertNull(innerBuffer.vectorsRoot.getVector("COLDATE").getObject(2));
 
     // Check stats generation
-    ChannelData result = innerBuffer.flush();
+    ChannelData<VectorSchemaRoot> result = innerBuffer.flush();
     Assert.assertEquals(3, result.getRowCount());
 
     Assert.assertEquals(
@@ -1335,7 +1342,8 @@ public void testE2ETime() {
     testE2ETimeHelper(this.channelOnErrorContinue);
   }
 
-  private void testE2ETimeHelper(SnowflakeStreamingIngestChannelInternal channel) {
+  private void testE2ETimeHelper(
+      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
     ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
 
     ColumnMetadata colTimeSB4 = new ColumnMetadata();
@@ -1380,7 +1388,7 @@ private void testE2ETimeHelper(SnowflakeStreamingIngestChannelInternal channel)
     Assert.assertNull(innerBuffer.vectorsRoot.getVector("COLTIMESB8").getObject(2));
 
     // Check stats generation
-    ChannelData result = innerBuffer.flush();
+    ChannelData<VectorSchemaRoot> result = innerBuffer.flush();
     Assert.assertEquals(3, result.getRowCount());
 
     Assert.assertEquals(
@@ -1404,7 +1412,8 @@ public void testNullableCheck() {
     testNullableCheckHelper(this.channelOnErrorAbort);
   }
 
-  private void testNullableCheckHelper(SnowflakeStreamingIngestChannelInternal channel) {
+  private void testNullableCheckHelper(
+      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
     ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
 
     ColumnMetadata colBoolean = new ColumnMetadata();
@@ -1444,7 +1453,8 @@ public void testMissingColumnCheck() {
     testMissingColumnCheckHelper(this.channelOnErrorAbort);
   }
 
-  private void testMissingColumnCheckHelper(SnowflakeStreamingIngestChannelInternal channel) {
+  private void testMissingColumnCheckHelper(
+      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
     ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
 
     ColumnMetadata colBoolean = new ColumnMetadata();
@@ -1518,7 +1528,8 @@ public void testE2EBoolean() {
     testE2EBooleanHelper(this.channelOnErrorAbort);
   }
 
-  private void testE2EBooleanHelper(SnowflakeStreamingIngestChannelInternal channel) {
+  private void testE2EBooleanHelper(
+      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
     ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
 
     ColumnMetadata colBoolean = new ColumnMetadata();
@@ -1550,7 +1561,7 @@ private void testE2EBooleanHelper(SnowflakeStreamingIngestChannelInternal channe
     Assert.assertNull(innerBuffer.vectorsRoot.getVector("COLBOOLEAN").getObject(2));
 
     // Check stats generation
-    ChannelData result = innerBuffer.flush();
+    ChannelData<VectorSchemaRoot> result = innerBuffer.flush();
     Assert.assertEquals(3, result.getRowCount());
 
     Assert.assertEquals(
@@ -1566,7 +1577,8 @@ public void testE2EBinary() {
     testE2EBinaryHelper(this.channelOnErrorContinue);
   }
 
-  private void testE2EBinaryHelper(SnowflakeStreamingIngestChannelInternal channel) {
+  private void testE2EBinaryHelper(
+      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
     ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
 
     ColumnMetadata colBinary = new ColumnMetadata();
@@ -1605,7 +1617,7 @@ private void testE2EBinaryHelper(SnowflakeStreamingIngestChannelInternal channel
     Assert.assertNull(innerBuffer.vectorsRoot.getVector("COLBINARY").getObject(2));
 
     // Check stats generation
-    ChannelData result = innerBuffer.flush();
+    ChannelData<VectorSchemaRoot> result = innerBuffer.flush();
 
     Assert.assertEquals(3, result.getRowCount());
     Assert.assertEquals(11L, result.getColumnEps().get("COLBINARY").getCurrentMaxLength());
@@ -1622,7 +1634,8 @@ public void testE2EReal() {
     testE2ERealHelper(this.channelOnErrorAbort);
   }
 
-  private void testE2ERealHelper(SnowflakeStreamingIngestChannelInternal channel) {
+  private void testE2ERealHelper(
+      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
     ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
 
     ColumnMetadata colReal = new ColumnMetadata();
@@ -1653,7 +1666,7 @@ private void testE2ERealHelper(SnowflakeStreamingIngestChannelInternal channel)
     Assert.assertNull(innerBuffer.vectorsRoot.getVector("COLREAL").getObject(2));
 
     // Check stats generation
-    ChannelData result = innerBuffer.flush();
+    ChannelData<VectorSchemaRoot> result = innerBuffer.flush();
 
     Assert.assertEquals(3, result.getRowCount());
     Assert.assertEquals(
@@ -1734,8 +1747,8 @@ public void testOnErrorAbortFailures() {
     Assert.assertNull(innerBuffer.tempStatsMap.get("COLDECIMAL").getCurrentMaxIntValue());
     Assert.assertNull(innerBuffer.tempStatsMap.get("COLDECIMAL").getCurrentMinIntValue());
 
-    ChannelData data = innerBuffer.flush();
-    Assert.assertEquals(3, data.getVectors().getRowCount());
+    ChannelData<VectorSchemaRoot> data = innerBuffer.flush();
+    Assert.assertEquals(3, data.getRowCount());
     Assert.assertEquals(0, innerBuffer.rowCount);
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 0857dcc76..82ca97f94 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -36,6 +36,7 @@
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.SnowflakeURL;
 import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.vector.VectorSchemaRoot;
 import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -50,8 +51,8 @@ public void testChannelFactoryNullFields() {
     String tableName = "TABLE";
     Long channelSequencer = 0L;
     Long rowSequencer = 0L;
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal("client");
+    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
+        new SnowflakeStreamingIngestClientInternal<>("client");
 
     Object[] fields =
         new Object[] {name, dbName, schemaName, tableName, channelSequencer, rowSequencer, client};
@@ -499,7 +500,7 @@ public void testInsertRow() {
     row.put("col", 1);
 
     // Get data before insert to verify that there is no row (data should be null)
-    ChannelData data = channel.getData();
+    ChannelData<VectorSchemaRoot> data = channel.getData();
     Assert.assertNull(data);
 
     InsertValidationResponse response = channel.insertRow(row, "1");
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index f651cba10..345e968aa 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -56,6 +56,7 @@
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.SnowflakeURL;
 import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.vector.VectorSchemaRoot;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -65,17 +66,17 @@
 public class SnowflakeStreamingIngestClientTest {
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
-  SnowflakeStreamingIngestChannelInternal channel1;
-  SnowflakeStreamingIngestChannelInternal channel2;
-  SnowflakeStreamingIngestChannelInternal channel3;
-  SnowflakeStreamingIngestChannelInternal channel4;
+  SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel1;
+  SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel2;
+  SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel3;
+  SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel4;
 
   @Before
   public void setup() {
     objectMapper.setVisibility(PropertyAccessor.GETTER, JsonAutoDetect.Visibility.ANY);
     objectMapper.setVisibility(PropertyAccessor.SETTER, JsonAutoDetect.Visibility.ANY);
     channel1 =
-        new SnowflakeStreamingIngestChannelInternal(
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel1",
             "db",
             "schemaName",
@@ -89,7 +90,7 @@ public void setup() {
             OpenChannelRequest.OnErrorOption.CONTINUE,
             true);
     channel2 =
-        new SnowflakeStreamingIngestChannelInternal(
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel2",
             "db",
             "schemaName",
@@ -103,7 +104,7 @@ public void setup() {
             OpenChannelRequest.OnErrorOption.CONTINUE,
             true);
     channel3 =
-        new SnowflakeStreamingIngestChannelInternal(
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel3",
             "db",
             "schemaName",
@@ -117,7 +118,7 @@ public void setup() {
             OpenChannelRequest.OnErrorOption.CONTINUE,
             true);
     channel4 =
-        new SnowflakeStreamingIngestChannelInternal(
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel4",
             "db",
             "schemaName",
@@ -145,8 +146,8 @@ public void testConstructorParameters() throws Exception {
     Map<String, Object> parameterMap = new HashMap<>();
     parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_MAP_KEY, 321);
 
-    SnowflakeStreamingIngestClientInternal client =
-        (SnowflakeStreamingIngestClientInternal)
+    SnowflakeStreamingIngestClientInternal<?> client =
+        (SnowflakeStreamingIngestClientInternal<?>)
             SnowflakeStreamingIngestClientFactory.builder("client").setProperties(prop).build();
 
     Assert.assertEquals("client", client.getName());
@@ -167,8 +168,8 @@ public void testClientFactoryWithJmxMetrics() throws Exception {
     prop.put(PRIVATE_KEY, TestUtils.getPrivateKey());
     prop.put(ROLE, "role");
 
-    SnowflakeStreamingIngestClientInternal client =
-        (SnowflakeStreamingIngestClientInternal)
+    SnowflakeStreamingIngestClientInternal<?> client =
+        (SnowflakeStreamingIngestClientInternal<?>)
             SnowflakeStreamingIngestClientFactory.builder("client")
                 .setProperties(prop)
                 .setParameterOverrides(
@@ -323,8 +324,8 @@ public void testGetChannelsStatusWithRequest() throws Exception {
     RequestBuilder requestBuilder =
         Mockito.spy(
             new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair()));
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal(
+    SnowflakeStreamingIngestClientInternal<?> client =
+        new SnowflakeStreamingIngestClientInternal<>(
             "client",
             new SnowflakeURL("snowflake.dev.local:8082"),
             null,
@@ -333,8 +334,8 @@ public void testGetChannelsStatusWithRequest() throws Exception {
             requestBuilder,
             null);
 
-    SnowflakeStreamingIngestChannelInternal channel =
-        new SnowflakeStreamingIngestChannelInternal(
+    SnowflakeStreamingIngestChannelInternal<?> channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel",
             "db",
             "schemaName",
@@ -381,8 +382,8 @@ public void testGetChannelsStatusWithRequestError() throws Exception {
     RequestBuilder requestBuilder =
         Mockito.spy(
             new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair()));
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal(
+    SnowflakeStreamingIngestClientInternal<?> client =
+        new SnowflakeStreamingIngestClientInternal<>(
             "client",
             new SnowflakeURL("snowflake.dev.local:8082"),
             null,
@@ -391,8 +392,8 @@ public void testGetChannelsStatusWithRequestError() throws Exception {
             requestBuilder,
             null);
 
-    SnowflakeStreamingIngestChannelInternal channel =
-        new SnowflakeStreamingIngestChannelInternal(
+    SnowflakeStreamingIngestChannelInternal<?> channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel",
             "db",
             "schemaName",
@@ -429,8 +430,8 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
     RequestBuilder requestBuilder =
         new RequestBuilder(url, prop.get(USER).toString(), keyPair, null, null);
 
-    SnowflakeStreamingIngestChannelInternal channel =
-        new SnowflakeStreamingIngestChannelInternal(
+    SnowflakeStreamingIngestChannelInternal<?> channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel",
             "db",
             "schemaName",
@@ -453,7 +454,7 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
 
     Map<String, RowBufferStats> columnEps = new HashMap<>();
     columnEps.put("column", new RowBufferStats());
-    EpInfo epInfo = ArrowRowBuffer.buildEpInfoFromStats(1, columnEps);
+    EpInfo epInfo = AbstractRowBuffer.buildEpInfoFromStats(1, columnEps);
 
     ChunkMetadata chunkMetadata =
         ChunkMetadata.builder()
@@ -495,7 +496,7 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
   private Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> getRetryBlobMetadata() {
     Map<String, RowBufferStats> columnEps = new HashMap<>();
     columnEps.put("column", new RowBufferStats());
-    EpInfo epInfo = ArrowRowBuffer.buildEpInfoFromStats(1, columnEps);
+    EpInfo epInfo = AbstractRowBuffer.buildEpInfoFromStats(1, columnEps);
 
     ChannelMetadata channelMetadata1 =
         ChannelMetadata.builder()
@@ -596,8 +597,8 @@ public void testGetRetryBlobs() throws Exception {
     RequestBuilder requestBuilder =
         new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair());
 
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal(
+    SnowflakeStreamingIngestClientInternal<?> client =
+        new SnowflakeStreamingIngestClientInternal<>(
             "client",
             new SnowflakeURL("snowflake.dev.local:8082"),
             null,
@@ -637,8 +638,8 @@ public void testRegisterBlobErrorResponse() throws Exception {
 
     RequestBuilder requestBuilder =
         new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair());
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal(
+    SnowflakeStreamingIngestClientInternal<?> client =
+        new SnowflakeStreamingIngestClientInternal<>(
             "client",
             new SnowflakeURL("snowflake.dev.local:8082"),
             null,
@@ -685,8 +686,8 @@ public void testRegisterBlobSnowflakeInternalErrorResponse() throws Exception {
 
     RequestBuilder requestBuilder =
         new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair());
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal(
+    SnowflakeStreamingIngestClientInternal<?> client =
+        new SnowflakeStreamingIngestClientInternal<>(
             "client",
             new SnowflakeURL("snowflake.dev.local:8082"),
             null,
@@ -742,8 +743,8 @@ public void testRegisterBlobSuccessResponse() throws Exception {
 
     RequestBuilder requestBuilder =
         new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair());
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal(
+    SnowflakeStreamingIngestClientInternal<?> client =
+        new SnowflakeStreamingIngestClientInternal<>(
             "client",
             new SnowflakeURL("snowflake.dev.local:8082"),
             null,
@@ -826,8 +827,8 @@ public void testRegisterBlobsRetries() throws Exception {
     RequestBuilder requestBuilder =
         Mockito.spy(
             new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair()));
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal(
+    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
+        new SnowflakeStreamingIngestClientInternal<>(
             "client",
             new SnowflakeURL("snowflake.dev.local:8082"),
             null,
@@ -941,8 +942,8 @@ public void testRegisterBlobsRetriesSucceeds() throws Exception {
     RequestBuilder requestBuilder =
         Mockito.spy(
             new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair()));
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal(
+    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
+        new SnowflakeStreamingIngestClientInternal<>(
             "client",
             new SnowflakeURL("snowflake.dev.local:8082"),
             null,
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index ddd2fafd2..7642a591e 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -51,7 +51,7 @@ public class StreamingIngestIT {
 
   private Properties prop;
 
-  private SnowflakeStreamingIngestClientInternal client;
+  private SnowflakeStreamingIngestClientInternal<?> client;
   private Connection jdbcConnection;
   private String testDb;
 
@@ -83,7 +83,7 @@ public void beforeAll() throws Exception {
       prop.setProperty(ROLE, "ACCOUNTADMIN");
     }
     client =
-        (SnowflakeStreamingIngestClientInternal)
+        (SnowflakeStreamingIngestClientInternal<?>)
             SnowflakeStreamingIngestClientFactory.builder("client1").setProperties(prop).build();
   }
 
@@ -167,7 +167,7 @@ public void testParameterOverrides() throws Exception {
     parameterMap.put(ParameterProvider.INSERT_THROTTLE_INTERVAL_IN_MILLIS_MAP_KEY, 1L);
     parameterMap.put(ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS_MAP_KEY, true);
     client =
-        (SnowflakeStreamingIngestClientInternal)
+        (SnowflakeStreamingIngestClientInternal<?>)
             SnowflakeStreamingIngestClientFactory.builder("testParameterOverridesClient")
                 .setProperties(prop)
                 .setParameterOverrides(parameterMap)
@@ -699,11 +699,11 @@ public void testMultiThread() throws Exception {
    */
   @Test
   public void testTwoClientsOneChannel() throws Exception {
-    SnowflakeStreamingIngestClientInternal clientA =
-        (SnowflakeStreamingIngestClientInternal)
+    SnowflakeStreamingIngestClientInternal<?> clientA =
+        (SnowflakeStreamingIngestClientInternal<?>)
             SnowflakeStreamingIngestClientFactory.builder("clientA").setProperties(prop).build();
-    SnowflakeStreamingIngestClientInternal clientB =
-        (SnowflakeStreamingIngestClientInternal)
+    SnowflakeStreamingIngestClientInternal<?> clientB =
+        (SnowflakeStreamingIngestClientInternal<?>)
             SnowflakeStreamingIngestClientFactory.builder("clientB").setProperties(prop).build();
 
     OpenChannelRequest requestA =
@@ -956,7 +956,7 @@ private void createTableForInterleavedTest(String tableName) {
           .createStatement()
           .execute(
               String.format(
-                  "create or replace table %s (num NUMBER(38,0 ), str VARCHAR);", tableName));
+                  "create or replace table %s (num NUMBER(38,0), str VARCHAR);", tableName));
     } catch (SQLException e) {
       throw new RuntimeException("Cannot create table " + tableName, e);
     }

From f107eca983daee2f0e0385af9b889d48dd47a2b1 Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Wed, 10 Aug 2022 17:57:54 +0000
Subject: [PATCH 060/356] @snow SNOW-633432 Refactor tests to support various
 BDEC format versions

---
 .../streaming/internal/AbstractRowBuffer.java |   8 +-
 .../streaming/internal/ArrowRowBuffer.java    |  12 +
 .../streaming/internal/ChannelMetadata.java   |   2 +-
 .../streaming/internal/FlushService.java      |   2 +-
 ...nowflakeStreamingIngestChannelFactory.java |  18 +-
 ...owflakeStreamingIngestChannelInternal.java |  91 +-
 .../streaming/internal/ArrowBufferTest.java   | 475 ++++++++++
 .../streaming/internal/ChannelCacheTest.java  |  49 +-
 .../streaming/internal/FlushServiceTest.java  | 560 ++++++-----
 .../internal/RegisterServiceTest.java         |  45 +-
 .../streaming/internal/RowBufferTest.java     | 866 +++---------------
 .../SnowflakeStreamingIngestChannelTest.java  | 145 ++-
 .../SnowflakeStreamingIngestClientTest.java   | 121 ++-
 .../streaming/internal/StubChunkData.java     |   4 +
 14 files changed, 1223 insertions(+), 1175 deletions(-)
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/StubChunkData.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 3a81dd375..2adacd14d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -27,7 +27,7 @@
  * un-flushed rows, these rows will be converted to the underlying format implementation for faster
  * processing
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot}
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
 abstract class AbstractRowBuffer<T> implements RowBuffer<T> {
   private static final Logging logger = new Logging(AbstractRowBuffer.class);
@@ -431,6 +431,12 @@ void reset() {
   /** Get buffered data snapshot for later flushing. */
   abstract Optional<T> getSnapshot();
 
+  @VisibleForTesting
+  abstract Object getVectorValueAt(String column, int index);
+
+  @VisibleForTesting
+  abstract int getTempRowCount();
+
   /** Normalize the column name, given with the inserted row, to send to server side. */
   static String formatColumnName(String columnName) {
     Utils.assertStringNotNullOrEmpty("invalid column name", columnName);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index b118c94da..83f925ad8 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -772,4 +772,16 @@ private void insertNull(FieldVector vector, RowBufferStats stats, int curRowInde
   public Flusher<VectorSchemaRoot> createFlusher(Constants.BdecVersion bdecVersion) {
     return new ArrowFlusher(bdecVersion);
   }
+
+  @VisibleForTesting
+  @Override
+  Object getVectorValueAt(String column, int index) {
+    Object value = vectorsRoot.getVector(column).getObject(index);
+    return (value instanceof Text) ? new String(((Text) value).getBytes()) : value;
+  }
+
+  @VisibleForTesting
+  int getTempRowCount() {
+    return tempVectorsRoot.getRowCount();
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelMetadata.java
index 3086634cb..6d0479f60 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelMetadata.java
@@ -29,7 +29,7 @@ static class Builder {
     private Long rowSequencer;
     @Nullable private String offsetToken; // offset token could be null
 
-    Builder setOwningChannel(SnowflakeStreamingIngestChannelInternal channel) {
+    Builder setOwningChannel(SnowflakeStreamingIngestChannelInternal<?> channel) {
       this.channelName = channel.getName();
       this.clientSequencer = channel.getChannelSequencer();
       return this;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 59f221158..ba7c61b4c 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -601,7 +601,7 @@ String getFilePath(Calendar calendar, String clientPrefix) {
    *
    * @param blobData list of channels that belongs to the blob
    */
-  void invalidateAllChannelsInBlob(List<List<ChannelData<T>>> blobData) {
+  <CD> void invalidateAllChannelsInBlob(List<List<ChannelData<CD>>> blobData) {
     blobData.forEach(
         chunkData ->
             chunkData.forEach(
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java
index 9e5cd1b46..73184019d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java
@@ -6,6 +6,8 @@
 
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
 
 /** Builds a Streaming Ingest channel for a specific Streaming Ingest client */
 class SnowflakeStreamingIngestChannelFactory {
@@ -94,6 +96,7 @@ SnowflakeStreamingIngestChannelInternal<T> build() {
       Utils.assertStringNotNullOrEmpty("encryption key", this.encryptionKey);
       Utils.assertNotNull("encryption key_id", this.encryptionKeyId);
       Utils.assertNotNull("on_error option", this.onErrorOption);
+      BufferAllocator allocator = createBufferAllocator();
       return new SnowflakeStreamingIngestChannelInternal<>(
           this.name,
           this.dbName,
@@ -105,7 +108,20 @@ SnowflakeStreamingIngestChannelInternal<T> build() {
           this.owningClient,
           this.encryptionKey,
           this.encryptionKeyId,
-          this.onErrorOption);
+          this.onErrorOption,
+          this.owningClient.getParameterProvider().getBlobFormatVersion(),
+          allocator);
+    }
+
+    private BufferAllocator createBufferAllocator() {
+      return owningClient.isTestMode()
+          ? new RootAllocator()
+          : owningClient
+              .getAllocator()
+              .newChildAllocator(
+                  String.format("%s_%s", name, channelSequencer),
+                  0,
+                  owningClient.getAllocator().getLimit());
     }
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index b5ab6a811..89df6231a 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -17,6 +17,7 @@
 import net.snowflake.ingest.streaming.InsertValidationResponse;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.SFException;
@@ -65,9 +66,6 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
   // Data encryption key id
   private final Long encryptionKeyId;
 
-  // Indicates whether we're using it as of the any tests
-  private boolean isTestMode;
-
   // ON_ERROR option for this channel
   private final OpenChannelRequest.OnErrorOption onErrorOption;
 
@@ -82,8 +80,36 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
    * @param channelSequencer
    * @param rowSequencer
    * @param client
-   * @param isTestMode
    */
+  SnowflakeStreamingIngestChannelInternal(
+      String name,
+      String dbName,
+      String schemaName,
+      String tableName,
+      String offsetToken,
+      Long channelSequencer,
+      Long rowSequencer,
+      SnowflakeStreamingIngestClientInternal<T> client,
+      String encryptionKey,
+      Long encryptionKeyId,
+      OpenChannelRequest.OnErrorOption onErrorOption) {
+    this(
+        name,
+        dbName,
+        schemaName,
+        tableName,
+        offsetToken,
+        channelSequencer,
+        rowSequencer,
+        client,
+        encryptionKey,
+        encryptionKeyId,
+        onErrorOption,
+        client.getParameterProvider().getBlobFormatVersion(),
+        new RootAllocator());
+  }
+
+  /** Default constructor */
   SnowflakeStreamingIngestChannelInternal(
       String name,
       String dbName,
@@ -96,7 +122,8 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
       String encryptionKey,
       Long encryptionKeyId,
       OpenChannelRequest.OnErrorOption onErrorOption,
-      boolean isTestMode) {
+      Constants.BdecVersion bdecVersion,
+      BufferAllocator allocator) {
     this.channelName = name;
     this.dbName = dbName;
     this.schemaName = schemaName;
@@ -107,24 +134,15 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
     this.isValid = true;
     this.isClosed = false;
     this.owningClient = client;
-    this.isTestMode = isTestMode;
-    this.allocator =
-        isTestMode || owningClient.isTestMode()
-            ? new RootAllocator()
-            : this.owningClient
-                .getAllocator()
-                .newChildAllocator(
-                    String.format("%s_%s", name, channelSequencer),
-                    0,
-                    this.owningClient.getAllocator().getLimit());
-    this.rowBuffer = createRowBuffer(client);
+    this.allocator = allocator;
+    this.rowBuffer = createRowBuffer(bdecVersion);
     this.encryptionKey = encryptionKey;
     this.encryptionKeyId = encryptionKeyId;
     this.onErrorOption = onErrorOption;
     logger.logInfo("Channel={} created for table={}", this.channelName, this.tableName);
   }
 
-  private RowBuffer<T> createRowBuffer(SnowflakeStreamingIngestClientInternal<T> client) {
+  private RowBuffer<T> createRowBuffer(Constants.BdecVersion bdecVersion) {
     // TODO: The circular dependency SnowflakeStreamingIngestChannelInternal <-> RowBuffer
     // (SNOW-657667)
     // can be probably reconsidered
@@ -133,45 +151,6 @@ private RowBuffer<T> createRowBuffer(SnowflakeStreamingIngestClientInternal<T> c
         new ArrowRowBuffer((SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>) this);
   }
 
-  /**
-   * Default Constructor
-   *
-   * @param name
-   * @param dbName
-   * @param schemaName
-   * @param tableName
-   * @param offsetToken
-   * @param channelSequencer
-   * @param rowSequencer
-   * @param client
-   */
-  SnowflakeStreamingIngestChannelInternal(
-      String name,
-      String dbName,
-      String schemaName,
-      String tableName,
-      String offsetToken,
-      Long channelSequencer,
-      Long rowSequencer,
-      SnowflakeStreamingIngestClientInternal<T> client,
-      String encryptionKey,
-      Long encryptionKeyId,
-      OpenChannelRequest.OnErrorOption onErrorOption) {
-    this(
-        name,
-        dbName,
-        schemaName,
-        tableName,
-        offsetToken,
-        channelSequencer,
-        rowSequencer,
-        client,
-        encryptionKey,
-        encryptionKeyId,
-        onErrorOption,
-        false);
-  }
-
   /**
    * Get the fully qualified channel name
    *
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
new file mode 100644
index 000000000..e8228b9f3
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
@@ -0,0 +1,475 @@
+package net.snowflake.ingest.streaming.internal;
+
+import static net.snowflake.ingest.streaming.internal.ArrowRowBuffer.DECIMAL_BIT_WIDTH;
+
+import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import net.snowflake.ingest.streaming.InsertValidationResponse;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ArrowBufferTest {
+  private ArrowRowBuffer rowBufferOnErrorContinue;
+
+  @Before
+  public void setupRowBuffer() {
+    this.rowBufferOnErrorContinue = createTestBuffer(OpenChannelRequest.OnErrorOption.CONTINUE);
+    this.rowBufferOnErrorContinue.setupSchema(RowBufferTest.createSchema());
+  }
+
+  ArrowRowBuffer createTestBuffer(OpenChannelRequest.OnErrorOption onErrorOption) {
+    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
+        new SnowflakeStreamingIngestClientInternal<>("client");
+    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
+            "channel", "db", "schema", "table", "0", 0L, 0L, client, "key", 1234L, onErrorOption);
+    return new ArrowRowBuffer(channel);
+  }
+
+  @Test
+  public void testFieldNumberAfterFlush() {
+    String offsetToken = "1";
+    Map<String, Object> row = new HashMap<>();
+    row.put("colTinyInt", (byte) 1);
+    row.put("\"colTinyInt\"", (byte) 1);
+    row.put("colSmallInt", (short) 2);
+    row.put("colInt", 3);
+    row.put("colBigInt", 4L);
+    row.put("colDecimal", 1.23);
+    row.put("colChar", "2");
+
+    InsertValidationResponse response =
+        rowBufferOnErrorContinue.insertRows(Collections.singletonList(row), offsetToken);
+    Assert.assertFalse(response.hasErrors());
+
+    ChannelData<VectorSchemaRoot> data = rowBufferOnErrorContinue.flush();
+    Assert.assertEquals(7, data.getVectors().getFieldVectors().size());
+  }
+
+  @Test
+  public void buildFieldFixedSB1() {
+    // FIXED, SB1
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB1");
+    testCol.setNullable(true);
+    testCol.setLogicalType("FIXED");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
+    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+
+    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.TINYINT.getType());
+    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB1");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "FIXED");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("nullable"), "true");
+    Assert.assertTrue(result.getFieldType().isNullable());
+    Assert.assertEquals(result.getChildren().size(), 0);
+  }
+
+  @Test
+  public void buildFieldFixedSB2() {
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB2");
+    testCol.setNullable(false);
+    testCol.setLogicalType("FIXED");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
+    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+
+    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.SMALLINT.getType());
+    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB2");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "FIXED");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("nullable"), "false");
+    Assert.assertFalse(result.getFieldType().isNullable());
+    Assert.assertEquals(result.getChildren().size(), 0);
+  }
+
+  @Test
+  public void buildFieldFixedSB4() {
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB4");
+    testCol.setNullable(true);
+    testCol.setLogicalType("FIXED");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
+    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+
+    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.INT.getType());
+    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB4");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "FIXED");
+    Assert.assertTrue(result.getFieldType().isNullable());
+    Assert.assertEquals(result.getChildren().size(), 0);
+  }
+
+  @Test
+  public void buildFieldFixedSB8() {
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB8");
+    testCol.setNullable(true);
+    testCol.setLogicalType("FIXED");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
+    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+
+    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.BIGINT.getType());
+    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "FIXED");
+    Assert.assertTrue(result.getFieldType().isNullable());
+    Assert.assertEquals(result.getChildren().size(), 0);
+  }
+
+  @Test
+  public void buildFieldFixedSB16() {
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB16");
+    testCol.setNullable(true);
+    testCol.setLogicalType("FIXED");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
+    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+
+    ArrowType expectedType =
+        new ArrowType.Decimal(testCol.getPrecision(), testCol.getScale(), DECIMAL_BIT_WIDTH);
+
+    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals(result.getFieldType().getType(), expectedType);
+    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB16");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "FIXED");
+    Assert.assertTrue(result.getFieldType().isNullable());
+    Assert.assertEquals(result.getChildren().size(), 0);
+  }
+
+  @Test
+  public void buildFieldLobVariant() {
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("LOB");
+    testCol.setNullable(true);
+    testCol.setLogicalType("VARIANT");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
+    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+
+    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.VARCHAR.getType());
+    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "LOB");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "VARIANT");
+    Assert.assertTrue(result.getFieldType().isNullable());
+    Assert.assertEquals(result.getChildren().size(), 0);
+  }
+
+  @Test
+  public void buildFieldTimestampNtzSB8() {
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB8");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIMESTAMP_NTZ");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
+    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+
+    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.BIGINT.getType());
+    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIMESTAMP_NTZ");
+    Assert.assertTrue(result.getFieldType().isNullable());
+    Assert.assertEquals(result.getChildren().size(), 0);
+  }
+
+  @Test
+  public void buildFieldTimestampNtzSB16() {
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB16");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIMESTAMP_NTZ");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
+    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+
+    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.STRUCT.getType());
+    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB16");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIMESTAMP_NTZ");
+    Assert.assertTrue(result.getFieldType().isNullable());
+    Assert.assertEquals(result.getChildren().size(), 2);
+    Assert.assertEquals(
+        result.getChildren().get(0).getFieldType().getType(), Types.MinorType.BIGINT.getType());
+    Assert.assertEquals(
+        result.getChildren().get(1).getFieldType().getType(), Types.MinorType.INT.getType());
+  }
+
+  @Test
+  public void buildFieldTimestampTzSB8() {
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB8");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIMESTAMP_TZ");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
+    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+
+    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.STRUCT.getType());
+    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIMESTAMP_TZ");
+    Assert.assertTrue(result.getFieldType().isNullable());
+    Assert.assertEquals(result.getChildren().size(), 2);
+    Assert.assertEquals(
+        result.getChildren().get(0).getFieldType().getType(), Types.MinorType.BIGINT.getType());
+    Assert.assertEquals(
+        result.getChildren().get(1).getFieldType().getType(), Types.MinorType.INT.getType());
+  }
+
+  @Test
+  public void buildFieldTimestampTzSB16() {
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB16");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIMESTAMP_TZ");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
+    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+
+    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.STRUCT.getType());
+    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB16");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIMESTAMP_TZ");
+    Assert.assertTrue(result.getFieldType().isNullable());
+    Assert.assertEquals(result.getChildren().size(), 3);
+    Assert.assertEquals(
+        result.getChildren().get(0).getFieldType().getType(), Types.MinorType.BIGINT.getType());
+    Assert.assertEquals(
+        result.getChildren().get(1).getFieldType().getType(), Types.MinorType.INT.getType());
+    Assert.assertEquals(
+        result.getChildren().get(2).getFieldType().getType(), Types.MinorType.INT.getType());
+  }
+
+  @Test
+  public void buildFieldDate() {
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB8");
+    testCol.setNullable(true);
+    testCol.setLogicalType("DATE");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
+    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+
+    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.DATEDAY.getType());
+    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "DATE");
+    Assert.assertTrue(result.getFieldType().isNullable());
+    Assert.assertEquals(result.getChildren().size(), 0);
+  }
+
+  @Test
+  public void buildFieldTimeSB4() {
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB4");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIME");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
+    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+
+    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.INT.getType());
+    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB4");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIME");
+    Assert.assertTrue(result.getFieldType().isNullable());
+    Assert.assertEquals(result.getChildren().size(), 0);
+  }
+
+  @Test
+  public void buildFieldTimeSB8() {
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB8");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIME");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
+    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+
+    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.BIGINT.getType());
+    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIME");
+    Assert.assertTrue(result.getFieldType().isNullable());
+    Assert.assertEquals(result.getChildren().size(), 0);
+  }
+
+  @Test
+  public void buildFieldBoolean() {
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("BINARY");
+    testCol.setNullable(true);
+    testCol.setLogicalType("BOOLEAN");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
+    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+
+    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.BIT.getType());
+    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "BINARY");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "BOOLEAN");
+    Assert.assertTrue(result.getFieldType().isNullable());
+    Assert.assertEquals(result.getChildren().size(), 0);
+  }
+
+  @Test
+  public void buildFieldRealSB16() {
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB16");
+    testCol.setNullable(true);
+    testCol.setLogicalType("REAL");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
+    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+
+    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.FLOAT8.getType());
+    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB16");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
+    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "REAL");
+    Assert.assertTrue(result.getFieldType().isNullable());
+    Assert.assertEquals(result.getChildren().size(), 0);
+  }
+
+  @Test
+  public void testArrowE2ETimestampLTZ() {
+    testArrowE2ETimestampLTZHelper(OpenChannelRequest.OnErrorOption.ABORT);
+    testArrowE2ETimestampLTZHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+  }
+
+  private void testArrowE2ETimestampLTZHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
+    ArrowRowBuffer innerBuffer = createTestBuffer(onErrorOption);
+
+    ColumnMetadata colTimestampLtzSB8 = new ColumnMetadata();
+    colTimestampLtzSB8.setName("COLTIMESTAMPLTZ_SB8");
+    colTimestampLtzSB8.setPhysicalType("SB8");
+    colTimestampLtzSB8.setNullable(false);
+    colTimestampLtzSB8.setLogicalType("TIMESTAMP_LTZ");
+    colTimestampLtzSB8.setScale(0);
+
+    ColumnMetadata colTimestampLtzSB16 = new ColumnMetadata();
+    colTimestampLtzSB16.setName("COLTIMESTAMPLTZ_SB16");
+    colTimestampLtzSB16.setPhysicalType("SB16");
+    colTimestampLtzSB16.setNullable(false);
+    colTimestampLtzSB16.setLogicalType("TIMESTAMP_LTZ");
+    colTimestampLtzSB16.setScale(9);
+
+    innerBuffer.setupSchema(Arrays.asList(colTimestampLtzSB8, colTimestampLtzSB16));
+
+    Map<String, Object> row = new HashMap<>();
+    row.put("COLTIMESTAMPLTZ_SB8", "1621899220");
+    row.put("COLTIMESTAMPLTZ_SB16", new BigDecimal("1621899220.123456789"));
+
+    InsertValidationResponse response =
+        innerBuffer.insertRows(Collections.singletonList(row), null);
+    Assert.assertFalse(response.hasErrors());
+    Assert.assertEquals(
+        1621899220L, innerBuffer.vectorsRoot.getVector("COLTIMESTAMPLTZ_SB8").getObject(0));
+    Assert.assertEquals(
+        "epoch",
+        innerBuffer
+            .vectorsRoot
+            .getVector("COLTIMESTAMPLTZ_SB16")
+            .getChildrenFromFields()
+            .get(0)
+            .getName());
+    Assert.assertEquals(
+        1621899220L,
+        innerBuffer
+            .vectorsRoot
+            .getVector("COLTIMESTAMPLTZ_SB16")
+            .getChildrenFromFields()
+            .get(0)
+            .getObject(0));
+    Assert.assertEquals(
+        "fraction",
+        innerBuffer
+            .vectorsRoot
+            .getVector("COLTIMESTAMPLTZ_SB16")
+            .getChildrenFromFields()
+            .get(1)
+            .getName());
+    Assert.assertEquals(
+        123456789,
+        innerBuffer
+            .vectorsRoot
+            .getVector("COLTIMESTAMPLTZ_SB16")
+            .getChildrenFromFields()
+            .get(1)
+            .getObject(0));
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java
index ba909d62d..7354a0f0d 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java
@@ -4,17 +4,16 @@
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
-import org.apache.arrow.vector.VectorSchemaRoot;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
 public class ChannelCacheTest {
-  ChannelCache<VectorSchemaRoot> cache;
-  SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client;
-  SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel1;
-  SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel2;
-  SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel3;
+  ChannelCache<StubChunkData> cache;
+  SnowflakeStreamingIngestClientInternal<StubChunkData> client;
+  SnowflakeStreamingIngestChannelInternal<StubChunkData> channel1;
+  SnowflakeStreamingIngestChannelInternal<StubChunkData> channel2;
+  SnowflakeStreamingIngestChannelInternal<StubChunkData> channel3;
   String dbName = "db";
   String schemaName = "schema";
   String table1Name = "table1";
@@ -36,8 +35,7 @@ public void setup() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
     channel2 =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel2",
@@ -50,8 +48,7 @@ public void setup() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
     channel3 =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel3",
@@ -64,8 +61,7 @@ public void setup() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
     cache.addChannel(channel1);
     cache.addChannel(channel2);
     cache.addChannel(channel3);
@@ -76,9 +72,9 @@ public void testAddChannel() {
     String channelName = "channel";
     String tableName = "table";
 
-    ChannelCache<VectorSchemaRoot> cache = new ChannelCache<>();
+    ChannelCache<StubChunkData> cache = new ChannelCache<>();
     Assert.assertEquals(0, cache.getSize());
-    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel =
+    SnowflakeStreamingIngestChannelInternal<StubChunkData> channel =
         new SnowflakeStreamingIngestChannelInternal<>(
             channelName,
             dbName,
@@ -90,13 +86,12 @@ public void testAddChannel() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
     cache.addChannel(channel);
     Assert.assertEquals(1, cache.getSize());
     Assert.assertTrue(channel == cache.iterator().next().getValue().get(channelName));
 
-    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channelDup =
+    SnowflakeStreamingIngestChannelInternal<StubChunkData> channelDup =
         new SnowflakeStreamingIngestChannelInternal<>(
             channelName,
             dbName,
@@ -108,14 +103,13 @@ public void testAddChannel() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
     cache.addChannel(channelDup);
     // The old channel should be invalid now
     Assert.assertTrue(!channel.isValid());
     Assert.assertTrue(channelDup.isValid());
     Assert.assertEquals(1, cache.getSize());
-    ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>> channels =
+    ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<StubChunkData>> channels =
         cache.iterator().next().getValue();
     Assert.assertEquals(1, channels.size());
     Assert.assertTrue(channelDup == channels.get(channelName));
@@ -128,16 +122,15 @@ public void testIterator() {
     Iterator<
             Map.Entry<
                 String,
-                ConcurrentHashMap<
-                    String, SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>>>>
+                ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<StubChunkData>>>>
         iter = cache.iterator();
     Map.Entry<
             String,
-            ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>>>
+            ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<StubChunkData>>>
         firstTable = iter.next();
     Map.Entry<
             String,
-            ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>>>
+            ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<StubChunkData>>>
         secondTable = iter.next();
     Assert.assertFalse(iter.hasNext());
     if (firstTable.getKey().equals(channel1.getFullyQualifiedTableName())) {
@@ -159,8 +152,7 @@ public void testCloseAllChannels() {
     Iterator<
             Map.Entry<
                 String,
-                ConcurrentHashMap<
-                    String, SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>>>>
+                ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<StubChunkData>>>>
         iter = cache.iterator();
     while (iter.hasNext()) {
       for (SnowflakeStreamingIngestChannelInternal<?> channel : iter.next().getValue().values()) {
@@ -179,7 +171,7 @@ public void testRemoveChannel() {
     cache.removeChannelIfSequencersMatch(channel2);
     Assert.assertEquals(1, cache.getSize());
 
-    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel3Dup =
+    SnowflakeStreamingIngestChannelInternal<StubChunkData> channel3Dup =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel3",
             dbName,
@@ -191,8 +183,7 @@ public void testRemoveChannel() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
     cache.removeChannelIfSequencersMatch(channel3Dup);
     // Verify that remove the same channel with a different channel sequencer is a no op
     Assert.assertEquals(1, cache.getSize());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 58746b8f7..2df8a1b19 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -23,6 +23,8 @@
 import java.util.Arrays;
 import java.util.Base64;
 import java.util.Calendar;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -31,7 +33,6 @@
 import javax.crypto.BadPaddingException;
 import javax.crypto.IllegalBlockSizeException;
 import javax.crypto.NoSuchPaddingException;
-import net.snowflake.client.jdbc.SnowflakeConnectionV1;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.utils.Constants;
@@ -41,93 +42,320 @@
 import net.snowflake.ingest.utils.SFException;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.IntVector;
-import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.util.Text;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
+@RunWith(Parameterized.class)
 public class FlushServiceTest {
-  private SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client;
-  private ChannelCache<VectorSchemaRoot> channelCache;
-  private SnowflakeConnectionV1 conn;
-  private SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel1;
-  private SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel2;
-  private SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel3;
-  private StreamingIngestStage stage;
+  @Parameterized.Parameters(name = "{0}")
+  public static Collection<Object[]> testContextFactory() {
+    return Arrays.asList(new Object[][] {{ArrowTestContext.createFactory()}});
+  }
+
+  public FlushServiceTest(TestContextFactory<?> testContextFactory) {
+    this.testContextFactory = testContextFactory;
+  }
+
+  private abstract static class TestContextFactory<T> {
+    private final String name;
+
+    TestContextFactory(String name) {
+      this.name = name;
+    }
+
+    abstract TestContext<T> create();
 
-  private final BufferAllocator allocator = new RootAllocator();
+    @Override
+    public String toString() {
+      return name;
+    }
+  }
+
+  private abstract static class TestContext<T> implements AutoCloseable {
+    SnowflakeStreamingIngestClientInternal<T> client;
+    ChannelCache<T> channelCache;
+    final Map<String, SnowflakeStreamingIngestChannelInternal<T>> channels = new HashMap<>();
+    FlushService<T> flushService;
+    StreamingIngestStage stage;
+    ParameterProvider parameterProvider;
+
+    final List<ChannelData<T>> channelData = new ArrayList<>();
+
+    TestContext() {
+      stage = Mockito.mock(StreamingIngestStage.class);
+      Mockito.when(stage.getClientPrefix()).thenReturn("client_prefix");
+      parameterProvider = new ParameterProvider();
+      client = Mockito.mock(SnowflakeStreamingIngestClientInternal.class);
+      Mockito.when(client.getParameterProvider()).thenReturn(parameterProvider);
+      channelCache = new ChannelCache<>();
+      Mockito.when(client.getChannelCache()).thenReturn(channelCache);
+      flushService = Mockito.spy(new FlushService<>(client, channelCache, stage, false));
+    }
+
+    ChannelData<T> flushChannel(String name) {
+      ChannelData<T> channelData = channels.get(name).getRowBuffer().flush();
+      this.channelData.add(channelData);
+      return channelData;
+    }
+
+    BlobMetadata buildAndUpload() throws Exception {
+      List<List<ChannelData<T>>> blobData = Collections.singletonList(channelData);
+      return flushService.buildAndUpload("file_name", blobData);
+    }
+
+    abstract SnowflakeStreamingIngestChannelInternal<T> createChannel(
+        String name,
+        String dbName,
+        String schemaName,
+        String tableName,
+        String offsetToken,
+        Long channelSequencer,
+        Long rowSequencer,
+        String encryptionKey,
+        Long encryptionKeyId,
+        OpenChannelRequest.OnErrorOption onErrorOption);
+
+    ChannelBuilder channelBuilder(String name) {
+      return new ChannelBuilder(name);
+    }
+
+    class ChannelBuilder {
+      private final String name;
+      private String dbName = "db1";
+      private String schemaName = "schema1";
+      private String tableName = "table1";
+      private String offsetToken = "offset1";
+      private Long channelSequencer = 0L;
+      private Long rowSequencer = 0L;
+      private String encryptionKey = "key";
+      private Long encryptionKeyId = 0L;
+      private OpenChannelRequest.OnErrorOption onErrorOption =
+          OpenChannelRequest.OnErrorOption.CONTINUE;
+
+      private ChannelBuilder(String name) {
+        this.name = name;
+      }
+
+      ChannelBuilder setDBName(String dbName) {
+        this.dbName = dbName;
+        return this;
+      }
+
+      ChannelBuilder setSchemaName(String schemaName) {
+        this.schemaName = schemaName;
+        return this;
+      }
+
+      ChannelBuilder setTableName(String tableName) {
+        this.tableName = tableName;
+        return this;
+      }
+
+      ChannelBuilder setOffsetToken(String offsetToken) {
+        this.offsetToken = offsetToken;
+        return this;
+      }
+
+      ChannelBuilder setChannelSequencer(Long sequencer) {
+        this.channelSequencer = sequencer;
+        return this;
+      }
+
+      ChannelBuilder setRowSequencer(Long sequencer) {
+        this.rowSequencer = sequencer;
+        return this;
+      }
+
+      ChannelBuilder setEncryptionKey(String encryptionKey) {
+        this.encryptionKey = encryptionKey;
+        return this;
+      }
+
+      ChannelBuilder setEncryptionKeyId(Long encryptionKeyId) {
+        this.encryptionKeyId = encryptionKeyId;
+        return this;
+      }
+
+      ChannelBuilder setOnErrorOption(OpenChannelRequest.OnErrorOption onErrorOption) {
+        this.onErrorOption = onErrorOption;
+        return this;
+      }
+
+      SnowflakeStreamingIngestChannelInternal<T> buildAndAdd() {
+        SnowflakeStreamingIngestChannelInternal<T> channel =
+            createChannel(
+                name,
+                dbName,
+                schemaName,
+                tableName,
+                offsetToken,
+                channelSequencer,
+                rowSequencer,
+                encryptionKey,
+                encryptionKeyId,
+                onErrorOption);
+        channels.put(name, channel);
+        channelCache.addChannel(channel);
+        return channel;
+      }
+    }
+  }
+
+  private static class RowSetBuilder {
+    private final List<Map<String, Object>> rows = new ArrayList<>();
+
+    private Map<String, Object> lastRow() {
+      return rows.get(rows.size() - 1);
+    }
+
+    RowSetBuilder addColumn(String column, Object value) {
+      lastRow().put(column, value);
+      return this;
+    }
+
+    RowSetBuilder newRow() {
+      if (rows.isEmpty() || !lastRow().isEmpty()) {
+        rows.add(new HashMap<>());
+      }
+      return this;
+    }
+
+    List<Map<String, Object>> build() {
+      return rows;
+    }
+
+    static RowSetBuilder newBuilder() {
+      return new RowSetBuilder().newRow();
+    }
+  }
+
+  private static class ArrowTestContext extends TestContext<VectorSchemaRoot> {
+    private final BufferAllocator allocator = new RootAllocator();
+
+    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> createChannel(
+        String name,
+        String dbName,
+        String schemaName,
+        String tableName,
+        String offsetToken,
+        Long channelSequencer,
+        Long rowSequencer,
+        String encryptionKey,
+        Long encryptionKeyId,
+        OpenChannelRequest.OnErrorOption onErrorOption) {
+      return new SnowflakeStreamingIngestChannelInternal<>(
+          name,
+          dbName,
+          schemaName,
+          tableName,
+          offsetToken,
+          channelSequencer,
+          rowSequencer,
+          client,
+          encryptionKey,
+          encryptionKeyId,
+          onErrorOption,
+          Constants.BdecVersion.ONE,
+          allocator);
+    }
+
+    @Override
+    public void close() {
+      try {
+        // Close allocator to make sure no memory leak
+        allocator.close();
+      } catch (Exception e) {
+        Assert.fail(String.format("Allocator close failure. Caused by %s", e.getMessage()));
+      }
+    }
+
+    static TestContextFactory<VectorSchemaRoot> createFactory() {
+      return new TestContextFactory<VectorSchemaRoot>("Arrow") {
+        @Override
+        TestContext<VectorSchemaRoot> create() {
+          return new ArrowTestContext();
+        }
+      };
+    }
+  }
+
+  TestContextFactory<?> testContextFactory;
+  TestContext<?> testContext;
 
   @Before
   public void setup() {
     java.security.Security.addProvider(new BouncyCastleProvider());
+    testContext = testContextFactory.create();
+  }
 
-    ParameterProvider parameterProvider = new ParameterProvider();
-    client = Mockito.mock(SnowflakeStreamingIngestClientInternal.class);
-    Mockito.when(client.getParameterProvider()).thenReturn(parameterProvider);
-    stage = Mockito.mock(StreamingIngestStage.class);
-    Mockito.when(stage.getClientPrefix()).thenReturn("client_prefix");
-    channelCache = new ChannelCache<>();
-    Mockito.when(client.getChannelCache()).thenReturn(channelCache);
-    conn = Mockito.mock(SnowflakeConnectionV1.class);
-    channel1 =
-        new SnowflakeStreamingIngestChannelInternal<>(
-            "channel1",
-            "db1",
-            "schema1",
-            "table1",
-            "offset1",
-            0L,
-            0L,
-            client,
-            "key",
-            1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+  private SnowflakeStreamingIngestChannelInternal<?> addChannel1() {
+    return testContext
+        .channelBuilder("channel1")
+        .setDBName("db1")
+        .setSchemaName("schema1")
+        .setTableName("table1")
+        .setOffsetToken("offset1")
+        .setChannelSequencer(0L)
+        .setRowSequencer(0L)
+        .buildAndAdd();
+  }
 
-    channel2 =
-        new SnowflakeStreamingIngestChannelInternal<>(
-            "channel2",
-            "db1",
-            "schema1",
-            "table1",
-            "offset2",
-            10L,
-            100L,
-            client,
-            "key",
-            1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+  private SnowflakeStreamingIngestChannelInternal<?> addChannel2() {
+    return testContext
+        .channelBuilder("channel2")
+        .setDBName("db1")
+        .setSchemaName("schema1")
+        .setTableName("table1")
+        .setOffsetToken("offset2")
+        .setChannelSequencer(10L)
+        .setRowSequencer(100L)
+        .buildAndAdd();
+  }
 
-    channel3 =
-        new SnowflakeStreamingIngestChannelInternal<>(
-            "channel3",
-            "db2",
-            "schema1",
-            "table2",
-            "offset3",
-            0L,
-            0L,
-            client,
-            "key",
-            1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
-    channelCache.addChannel(channel1);
-    channelCache.addChannel(channel2);
-    channelCache.addChannel(channel3);
+  private SnowflakeStreamingIngestChannelInternal<?> addChannel3() {
+    return testContext
+        .channelBuilder("channel3")
+        .setDBName("db2")
+        .setSchemaName("schema1")
+        .setTableName("table2")
+        .setOffsetToken("offset3")
+        .setChannelSequencer(0L)
+        .setRowSequencer(0L)
+        .buildAndAdd();
+  }
+
+  private static ColumnMetadata createTestIntegerColumn() {
+    ColumnMetadata colInt = new ColumnMetadata();
+    colInt.setName("COLINT");
+    colInt.setPhysicalType("SB4");
+    colInt.setNullable(true);
+    colInt.setLogicalType("FIXED");
+    colInt.setScale(0);
+    return colInt;
+  }
+
+  private static ColumnMetadata createTestTextColumn() {
+    ColumnMetadata colChar = new ColumnMetadata();
+    colChar.setName("COLCHAR");
+    colChar.setPhysicalType("LOB");
+    colChar.setNullable(true);
+    colChar.setLogicalType("TEXT");
+    colChar.setByteLength(14);
+    colChar.setLength(11);
+    colChar.setScale(0);
+    colChar.setCollation("en-ci");
+    return colChar;
   }
 
   @Test
   public void testGetFilePath() {
-    FlushService<VectorSchemaRoot> flushService =
-        new FlushService<>(client, channelCache, stage, false);
+    FlushService<?> flushService = testContext.flushService;
     Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
     String clientPrefix = "honk";
     String outputString = flushService.getFilePath(calendar, clientPrefix);
@@ -160,8 +388,7 @@ public void testGetFilePath() {
 
   @Test
   public void testFlush() throws Exception {
-    FlushService<VectorSchemaRoot> flushService =
-        Mockito.spy(new FlushService<>(client, channelCache, stage, false));
+    FlushService<?> flushService = testContext.flushService;
 
     // Nothing to flush
     flushService.flush(false).get();
@@ -187,34 +414,29 @@ public void testFlush() throws Exception {
 
   @Test
   public void testBuildAndUpload() throws Exception {
-    FlushService<VectorSchemaRoot> flushService =
-        Mockito.spy(new FlushService<>(client, channelCache, stage, true));
-
-    List<List<ChannelData<VectorSchemaRoot>>> blobData = new ArrayList<>();
-    List<ChannelData<VectorSchemaRoot>> chunkData = new ArrayList<>();
-
-    // Construct fields
-    ChannelData<VectorSchemaRoot> channel1Data = new ChannelData<>();
-    ChannelData<VectorSchemaRoot> channel2Data = new ChannelData<>();
-
-    FieldVector vector1 = new VarCharVector("vector1", allocator);
-    FieldVector vector2 = new IntVector("vector2", allocator);
-    FieldVector vector3 = new VarCharVector("vector3", allocator);
-    FieldVector vector4 = new IntVector("vector4", allocator);
-    List<FieldVector> vectorList1 = new ArrayList<>();
-    vectorList1.add(vector1);
-    vectorList1.add(vector2);
-    List<FieldVector> vectorList2 = new ArrayList<>();
-    vectorList2.add(vector3);
-    vectorList2.add(vector4);
-
-    VectorSchemaRoot vectorRoot1 = new VectorSchemaRoot(vectorList1);
-    vectorRoot1.setRowCount(2);
-    VectorSchemaRoot vectorRoot2 = new VectorSchemaRoot(vectorList2);
-    vectorRoot2.setRowCount(1);
-
-    channel1Data.setVectors(vectorRoot1);
-    channel2Data.setVectors(vectorRoot2);
+    SnowflakeStreamingIngestChannelInternal<?> channel1 = addChannel1();
+    SnowflakeStreamingIngestChannelInternal<?> channel2 = addChannel2();
+
+    List<ColumnMetadata> schema = Arrays.asList(createTestIntegerColumn(), createTestTextColumn());
+    channel1.getRowBuffer().setupSchema(schema);
+    channel2.getRowBuffer().setupSchema(schema);
+
+    List<Map<String, Object>> rows1 =
+        RowSetBuilder.newBuilder()
+            .addColumn("COLINT", 11)
+            .addColumn("COLCHAR", "bob")
+            .newRow()
+            .addColumn("COLINT", 22)
+            .addColumn("COLCHAR", "bob")
+            .build();
+    List<Map<String, Object>> rows2 =
+        RowSetBuilder.newBuilder().addColumn("COLINT", null).addColumn("COLCHAR", "toby").build();
+
+    channel1.insertRows(rows1, "offset1");
+    channel2.insertRows(rows2, "offset2");
+
+    ChannelData<?> channel1Data = testContext.flushChannel(channel1.getName());
+    ChannelData<?> channel2Data = testContext.flushChannel(channel2.getName());
 
     Map<String, RowBufferStats> eps1 = new HashMap<>();
     Map<String, RowBufferStats> eps2 = new HashMap<>();
@@ -234,37 +456,12 @@ public void testBuildAndUpload() throws Exception {
     channel1Data.setColumnEps(eps1);
     channel2Data.setColumnEps(eps2);
 
-    chunkData.add(channel1Data);
-    chunkData.add(channel2Data);
-    blobData.add(chunkData);
-
-    // Populate test row data
-    ((VarCharVector) vector1).setSafe(0, new Text("alice"));
-    ((VarCharVector) vector1).setSafe(1, new Text("bob"));
-    ((IntVector) vector2).setSafe(0, 11);
-    ((IntVector) vector2).setSafe(1, 22);
-
-    ((VarCharVector) vector3).setSafe(0, new Text("toby"));
-    ((IntVector) vector4).setNull(0);
-
-    vector1.setValueCount(2);
-    vector2.setValueCount(2);
-    vector3.setValueCount(1);
-    vector4.setValueCount(1);
-
     channel1Data.setRowSequencer(0L);
-    channel1Data.setOffsetToken("offset1");
     channel1Data.setBufferSize(100);
-    channel1Data.setChannel(channel1);
-    channel1Data.setRowCount(2);
-
     channel2Data.setRowSequencer(10L);
-    channel2Data.setOffsetToken("offset2");
     channel2Data.setBufferSize(100);
-    channel2Data.setChannel(channel2);
-    channel2Data.setRowCount(1);
 
-    BlobMetadata blobMetadata = flushService.buildAndUpload("file_name", blobData);
+    BlobMetadata blobMetadata = testContext.buildAndUpload();
 
     EpInfo expectedChunkEpInfo =
         AbstractRowBuffer.buildEpInfoFromStats(
@@ -298,7 +495,7 @@ public void testBuildAndUpload() throws Exception {
     final ArgumentCaptor<byte[]> blobCaptor = ArgumentCaptor.forClass(byte[].class);
     final ArgumentCaptor<List<ChunkMetadata>> metadataCaptor = ArgumentCaptor.forClass(List.class);
 
-    Mockito.verify(flushService)
+    Mockito.verify(testContext.flushService)
         .upload(nameCaptor.capture(), blobCaptor.capture(), metadataCaptor.capture());
     Assert.assertEquals("file_name", nameCaptor.getValue());
 
@@ -339,67 +536,37 @@ public void testBuildAndUpload() throws Exception {
                 (int) (metadataResult.getChunkStartOffset() + metadataResult.getChunkLength())));
     Assert.assertEquals(md5, metadataResult.getChunkMD5());
 
-    try {
-      // Close allocator to make sure no memory leak
-      allocator.close();
-    } catch (Exception e) {
-      Assert.fail(String.format("Allocator close failure. Caused by %s", e.getMessage()));
-    }
+    testContext.close();
   }
 
   @Test
   public void testBuildErrors() throws Exception {
-    // build should error if we try to group channels for different tables together
-    FlushService<VectorSchemaRoot> flushService =
-        Mockito.spy(new FlushService<>(client, channelCache, stage, true));
-
-    List<List<ChannelData<VectorSchemaRoot>>> channelData = new ArrayList<>();
-    List<ChannelData<VectorSchemaRoot>> channelData1 = new ArrayList<>();
-
-    // Construct fields
-    ChannelData<VectorSchemaRoot> data1 = new ChannelData<>();
-    ChannelData<VectorSchemaRoot> data2 = new ChannelData<>();
+    SnowflakeStreamingIngestChannelInternal<?> channel1 = addChannel1();
+    SnowflakeStreamingIngestChannelInternal<?> channel3 = addChannel3();
 
-    FieldVector vector1 = new VarCharVector("vector1", allocator);
-    FieldVector vector3 = new IntVector("vector3", allocator);
-    vector1.allocateNew();
-    vector3.allocateNew();
-    List<FieldVector> vectorList1 = new ArrayList<>();
-    vectorList1.add(vector1);
-    List<FieldVector> vectorList2 = new ArrayList<>();
-    vectorList1.add(vector3);
+    List<ColumnMetadata> schema = Arrays.asList(createTestIntegerColumn(), createTestTextColumn());
+    channel1.getRowBuffer().setupSchema(schema);
+    channel3.getRowBuffer().setupSchema(schema);
 
-    VectorSchemaRoot vectorRoot1 = new VectorSchemaRoot(vectorList1);
-    vectorRoot1.setRowCount(2);
-    VectorSchemaRoot vectorRoot2 = new VectorSchemaRoot(vectorList2);
-    vectorRoot2.setRowCount(1);
+    List<Map<String, Object>> rows1 =
+        RowSetBuilder.newBuilder().addColumn("COLINT", 0).addColumn("COLCHAR", "alice").build();
+    List<Map<String, Object>> rows2 =
+        RowSetBuilder.newBuilder().addColumn("COLINT", 0).addColumn("COLCHAR", 111).build();
 
-    data1.setVectors(vectorRoot1);
-    data2.setVectors(vectorRoot2);
+    channel1.insertRows(rows1, "offset1");
+    channel3.insertRows(rows2, "offset2");
 
-    channelData1.add(data1);
-    channelData1.add(data2);
-    channelData.add(channelData1);
-
-    // Populate test row data
-    ((VarCharVector) vector1).setSafe(0, new Text("alice"));
-    ((IntVector) vector3).setSafe(0, 111);
-
-    vector1.setValueCount(2);
-    vector3.setValueCount(3);
+    ChannelData<?> data1 = testContext.flushChannel(channel1.getName());
+    ChannelData<?> data2 = testContext.flushChannel(channel3.getName());
 
     data1.setRowSequencer(0L);
-    data1.setOffsetToken("offset1");
     data1.setBufferSize(100);
-    data1.setChannel(channel1);
 
     data2.setRowSequencer(10L);
-    data2.setOffsetToken("offset3");
     data2.setBufferSize(100);
-    data2.setChannel(channel3);
 
     try {
-      flushService.buildAndUpload("file_name", channelData);
+      testContext.buildAndUpload();
       Assert.fail("Expected SFException");
     } catch (SFException err) {
       Assert.assertEquals(ErrorCode.INVALID_DATA_IN_CHUNK.getMessageCode(), err.getVendorCode());
@@ -407,15 +574,15 @@ public void testBuildErrors() throws Exception {
   }
 
   @Test
-  public void testInvalidateChannels() throws Exception {
+  public void testInvalidateChannels() {
     // Create a new Client in order to not interfere with other tests
-    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
+    SnowflakeStreamingIngestClientInternal<StubChunkData> client =
         Mockito.mock(SnowflakeStreamingIngestClientInternal.class);
     ParameterProvider parameterProvider = new ParameterProvider();
-    ChannelCache<VectorSchemaRoot> channelCache = new ChannelCache<>();
+    ChannelCache<StubChunkData> channelCache = new ChannelCache<>();
     Mockito.when(client.getChannelCache()).thenReturn(channelCache);
     Mockito.when(client.getParameterProvider()).thenReturn(parameterProvider);
-    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel1 =
+    SnowflakeStreamingIngestChannelInternal<StubChunkData> channel1 =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel1",
             "db1",
@@ -427,10 +594,9 @@ public void testInvalidateChannels() throws Exception {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
 
-    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel2 =
+    SnowflakeStreamingIngestChannelInternal<StubChunkData> channel2 =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel2",
             "db1",
@@ -442,42 +608,27 @@ public void testInvalidateChannels() throws Exception {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
 
     channelCache.addChannel(channel1);
     channelCache.addChannel(channel2);
 
-    List<List<ChannelData<VectorSchemaRoot>>> blobData = new ArrayList<>();
-    List<ChannelData<VectorSchemaRoot>> innerData = new ArrayList<>();
+    List<List<ChannelData<StubChunkData>>> blobData = new ArrayList<>();
+    List<ChannelData<StubChunkData>> innerData = new ArrayList<>();
     blobData.add(innerData);
 
-    ChannelData<VectorSchemaRoot> channel1Data = new ChannelData<>();
-    ChannelData<VectorSchemaRoot> channel2Data = new ChannelData<>();
+    ChannelData<StubChunkData> channel1Data = new ChannelData<>();
+    ChannelData<StubChunkData> channel2Data = new ChannelData<>();
     channel1Data.setChannel(channel1);
     channel2Data.setChannel(channel1);
 
-    FieldVector vector1 = new VarCharVector("vector1", allocator);
-    FieldVector vector2 = new IntVector("vector2", allocator);
-    FieldVector vector3 = new VarCharVector("vector3", allocator);
-    FieldVector vector4 = new IntVector("vector4", allocator);
-    List<FieldVector> vectorList1 = new ArrayList<>();
-    vectorList1.add(vector1);
-    vectorList1.add(vector2);
-    List<FieldVector> vectorList2 = new ArrayList<>();
-    vectorList2.add(vector3);
-    vectorList2.add(vector4);
-
-    VectorSchemaRoot vectorRoot1 = new VectorSchemaRoot(vectorList1);
-    VectorSchemaRoot vectorRoot2 = new VectorSchemaRoot(vectorList2);
-
-    channel1Data.setVectors(vectorRoot1);
-    channel2Data.setVectors(vectorRoot2);
+    channel1Data.setVectors(new StubChunkData());
+    channel2Data.setVectors(new StubChunkData());
     innerData.add(channel1Data);
     innerData.add(channel2Data);
 
-    FlushService<VectorSchemaRoot> flushService =
-        new FlushService<>(client, channelCache, stage, false);
+    FlushService<StubChunkData> flushService =
+        new FlushService<>(client, channelCache, testContext.stage, false);
     flushService.invalidateAllChannelsInBlob(blobData);
 
     Assert.assertFalse(channel1.isValid());
@@ -486,6 +637,8 @@ public void testInvalidateChannels() throws Exception {
 
   @Test
   public void testBlobBuilder() throws Exception {
+    SnowflakeStreamingIngestChannelInternal<?> channel1 = addChannel1();
+
     ObjectMapper mapper = new ObjectMapper();
     List<ChunkMetadata> chunksMetadataList = new ArrayList<>();
     List<byte[]> chunksDataList = new ArrayList<>();
@@ -516,7 +669,7 @@ public void testBlobBuilder() throws Exception {
             .setOwningTable(channel1)
             .setChunkStartOffset(0L)
             .setChunkLength(dataSize)
-            .setChannelList(Arrays.asList(channelMetadata))
+            .setChannelList(Collections.singletonList(channelMetadata))
             .setChunkMD5("md5")
             .setEncryptionKeyId(1234L)
             .setEpInfo(epInfo)
@@ -578,8 +731,7 @@ public void testBlobBuilder() throws Exception {
 
   @Test
   public void testShutDown() throws Exception {
-    FlushService<VectorSchemaRoot> flushService =
-        new FlushService<>(client, channelCache, stage, false);
+    FlushService<?> flushService = testContext.flushService;
 
     Assert.assertFalse(flushService.buildUploadWorkers.isShutdown());
     Assert.assertFalse(flushService.registerWorker.isShutdown());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
index 53f96808a..d2d2d3301 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
@@ -10,7 +10,6 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import net.snowflake.ingest.utils.Pair;
-import org.apache.arrow.vector.VectorSchemaRoot;
 import org.junit.Assert;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -18,16 +17,16 @@
 public class RegisterServiceTest {
 
   @Test
-  public void testRegisterService() throws Exception {
-    RegisterService<VectorSchemaRoot> rs = new RegisterService<>(null, true);
+  public void testRegisterService() {
+    RegisterService<StubChunkData> rs = new RegisterService<>(null, true);
 
-    Pair<FlushService.BlobData<VectorSchemaRoot>, CompletableFuture<BlobMetadata>> blobFuture =
+    Pair<FlushService.BlobData<StubChunkData>, CompletableFuture<BlobMetadata>> blobFuture =
         new Pair<>(
             new FlushService.BlobData<>("test", null),
             CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null)));
     rs.addBlobs(Collections.singletonList(blobFuture));
     Assert.assertEquals(1, rs.getBlobsList().size());
-    List<FlushService.BlobData<VectorSchemaRoot>> errorBlobs = rs.registerBlobs(null);
+    List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null);
     Assert.assertEquals(0, rs.getBlobsList().size());
     Assert.assertEquals(0, errorBlobs.size());
   }
@@ -43,22 +42,22 @@ public void testRegisterService() throws Exception {
    */
   @Test
   public void testRegisterServiceTimeoutException() throws Exception {
-    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
+    SnowflakeStreamingIngestClientInternal<StubChunkData> client =
         new SnowflakeStreamingIngestClientInternal<>("client");
-    RegisterService<VectorSchemaRoot> rs = new RegisterService<>(client, true);
+    RegisterService<StubChunkData> rs = new RegisterService<>(client, true);
 
-    Pair<FlushService.BlobData<VectorSchemaRoot>, CompletableFuture<BlobMetadata>> blobFuture1 =
+    Pair<FlushService.BlobData<StubChunkData>, CompletableFuture<BlobMetadata>> blobFuture1 =
         new Pair<>(
             new FlushService.BlobData<>("success", new ArrayList<>()),
             CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null)));
     CompletableFuture future = new CompletableFuture();
     future.completeExceptionally(new TimeoutException());
-    Pair<FlushService.BlobData<VectorSchemaRoot>, CompletableFuture<BlobMetadata>> blobFuture2 =
-        new Pair<>(new FlushService.BlobData<VectorSchemaRoot>("fail", new ArrayList<>()), future);
+    Pair<FlushService.BlobData<StubChunkData>, CompletableFuture<BlobMetadata>> blobFuture2 =
+        new Pair<>(new FlushService.BlobData<StubChunkData>("fail", new ArrayList<>()), future);
     rs.addBlobs(Arrays.asList(blobFuture1, blobFuture2));
     Assert.assertEquals(2, rs.getBlobsList().size());
     try {
-      List<FlushService.BlobData<VectorSchemaRoot>> errorBlobs = rs.registerBlobs(null);
+      List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null);
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
       Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
@@ -71,11 +70,11 @@ public void testRegisterServiceTimeoutException() throws Exception {
   @Ignore
   @Test
   public void testRegisterServiceTimeoutException_testRetries() throws Exception {
-    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
+    SnowflakeStreamingIngestClientInternal<StubChunkData> client =
         new SnowflakeStreamingIngestClientInternal<>("client");
-    RegisterService<VectorSchemaRoot> rs = new RegisterService<>(client, true);
+    RegisterService<StubChunkData> rs = new RegisterService<>(client, true);
 
-    Pair<FlushService.BlobData<VectorSchemaRoot>, CompletableFuture<BlobMetadata>> blobFuture1 =
+    Pair<FlushService.BlobData<StubChunkData>, CompletableFuture<BlobMetadata>> blobFuture1 =
         new Pair<>(
             new FlushService.BlobData<>("success", new ArrayList<>()),
             CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null)));
@@ -89,12 +88,12 @@ public void testRegisterServiceTimeoutException_testRetries() throws Exception {
           }
           return;
         });
-    Pair<FlushService.BlobData<VectorSchemaRoot>, CompletableFuture<BlobMetadata>> blobFuture2 =
-        new Pair<>(new FlushService.BlobData<VectorSchemaRoot>("fail", new ArrayList<>()), future);
+    Pair<FlushService.BlobData<StubChunkData>, CompletableFuture<BlobMetadata>> blobFuture2 =
+        new Pair<>(new FlushService.BlobData<StubChunkData>("fail", new ArrayList<>()), future);
     rs.addBlobs(Arrays.asList(blobFuture1, blobFuture2));
     Assert.assertEquals(2, rs.getBlobsList().size());
     try {
-      List<FlushService.BlobData<VectorSchemaRoot>> errorBlobs = rs.registerBlobs(null);
+      List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null);
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
       Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
@@ -105,18 +104,18 @@ public void testRegisterServiceTimeoutException_testRetries() throws Exception {
 
   @Test
   public void testRegisterServiceNonTimeoutException() {
-    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
+    SnowflakeStreamingIngestClientInternal<StubChunkData> client =
         new SnowflakeStreamingIngestClientInternal<>("client");
-    RegisterService<VectorSchemaRoot> rs = new RegisterService<>(client, true);
+    RegisterService<StubChunkData> rs = new RegisterService<>(client, true);
 
-    CompletableFuture future = new CompletableFuture();
+    CompletableFuture<BlobMetadata> future = new CompletableFuture<>();
     future.completeExceptionally(new IndexOutOfBoundsException());
-    Pair<FlushService.BlobData<VectorSchemaRoot>, CompletableFuture<BlobMetadata>> blobFuture =
-        new Pair<>(new FlushService.BlobData<VectorSchemaRoot>("fail", new ArrayList<>()), future);
+    Pair<FlushService.BlobData<StubChunkData>, CompletableFuture<BlobMetadata>> blobFuture =
+        new Pair<>(new FlushService.BlobData<>("fail", new ArrayList<>()), future);
     rs.addBlobs(Collections.singletonList(blobFuture));
     Assert.assertEquals(1, rs.getBlobsList().size());
     try {
-      List<FlushService.BlobData<VectorSchemaRoot>> errorBlobs = rs.registerBlobs(null);
+      List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null);
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
       Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index fcf896e40..4f2e12b86 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -1,76 +1,52 @@
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.streaming.internal.ArrowRowBuffer.DECIMAL_BIT_WIDTH;
-
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.SFException;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.types.Types;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.util.Text;
+import org.apache.arrow.memory.RootAllocator;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
+@RunWith(Parameterized.class)
 public class RowBufferTest {
-  private static final Logging logger = new Logging(RegisterService.class);
+  @Parameterized.Parameters(name = "{0}")
+  public static Collection<Object[]> bdecVersion() {
+    return Arrays.asList(new Object[][] {{"Arrow", Constants.BdecVersion.ONE}});
+  }
 
-  private ArrowRowBuffer rowBufferOnErrorContinue;
-  private SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channelOnErrorContinue;
+  private final Constants.BdecVersion bdecVersion;
+  private AbstractRowBuffer<?> rowBufferOnErrorContinue;
+  private AbstractRowBuffer<?> rowBufferOnErrorAbort;
 
-  private ArrowRowBuffer rowBufferOnErrorAbort;
-  private SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channelOnErrorAbort;
+  public RowBufferTest(@SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
+    this.bdecVersion = bdecVersion;
+  }
 
   @Before
   public void setupRowBuffer() {
-    // Create row buffer
-    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
-        new SnowflakeStreamingIngestClientInternal<>("client");
-    this.channelOnErrorContinue =
-        new SnowflakeStreamingIngestChannelInternal<>(
-            "channel",
-            "db",
-            "schema",
-            "table",
-            "0",
-            0L,
-            0L,
-            client,
-            "key",
-            1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
-    this.rowBufferOnErrorContinue = new ArrowRowBuffer(this.channelOnErrorContinue);
-
-    this.channelOnErrorAbort =
-        new SnowflakeStreamingIngestChannelInternal<>(
-            "channel",
-            "db",
-            "schema",
-            "table",
-            "0",
-            0L,
-            0L,
-            client,
-            "key",
-            1234L,
-            OpenChannelRequest.OnErrorOption.ABORT,
-            true);
-    this.rowBufferOnErrorAbort = new ArrowRowBuffer(this.channelOnErrorAbort);
+    this.rowBufferOnErrorContinue = createTestBuffer(OpenChannelRequest.OnErrorOption.CONTINUE);
+    this.rowBufferOnErrorAbort = createTestBuffer(OpenChannelRequest.OnErrorOption.ABORT);
+    List<ColumnMetadata> schema = createSchema();
+    this.rowBufferOnErrorContinue.setupSchema(schema);
+    this.rowBufferOnErrorAbort.setupSchema(schema);
+  }
 
+  static List<ColumnMetadata> createSchema() {
     ColumnMetadata colTinyIntCase = new ColumnMetadata();
     colTinyIntCase.setName("\"colTinyInt\"");
     colTinyIntCase.setPhysicalType("SB1");
@@ -124,13 +100,27 @@ public void setupRowBuffer() {
     colChar.setScale(0);
     colChar.setCollation("en-ci");
 
-    // Setup column fields and vectors
-    this.rowBufferOnErrorContinue.setupSchema(
-        Arrays.asList(
-            colTinyIntCase, colTinyInt, colSmallInt, colInt, colBigInt, colDecimal, colChar));
-    this.rowBufferOnErrorAbort.setupSchema(
-        Arrays.asList(
-            colTinyIntCase, colTinyInt, colSmallInt, colInt, colBigInt, colDecimal, colChar));
+    return Arrays.asList(
+        colTinyIntCase, colTinyInt, colSmallInt, colInt, colBigInt, colDecimal, colChar);
+  }
+
+  private AbstractRowBuffer<?> createTestBuffer(OpenChannelRequest.OnErrorOption onErrorOption) {
+    return (AbstractRowBuffer<?>)
+        new SnowflakeStreamingIngestChannelInternal<>(
+                "channel",
+                "db",
+                "schema",
+                "table",
+                "0",
+                0L,
+                0L,
+                new SnowflakeStreamingIngestClientInternal<>("client"),
+                "key",
+                1234L,
+                onErrorOption,
+                bdecVersion,
+                new RootAllocator())
+            .getRowBuffer();
   }
 
   @Test
@@ -146,7 +136,7 @@ public void buildFieldErrorStates() {
     testCol.setScale(0);
     testCol.setPrecision(4);
     try {
-      Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+      this.rowBufferOnErrorContinue.setupSchema(Collections.singletonList(testCol));
       Assert.fail("Expected error");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.UNKNOWN_DATA_TYPE.getMessageCode(), e.getVendorCode());
@@ -157,7 +147,7 @@ public void buildFieldErrorStates() {
     testCol.setPhysicalType("LOB");
     testCol.setLogicalType("FIXED");
     try {
-      Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+      this.rowBufferOnErrorContinue.setupSchema(Collections.singletonList(testCol));
       Assert.fail("Expected error");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.UNKNOWN_DATA_TYPE.getMessageCode(), e.getVendorCode());
@@ -168,7 +158,7 @@ public void buildFieldErrorStates() {
     testCol.setPhysicalType("SB2");
     testCol.setLogicalType("TIMESTAMP_NTZ");
     try {
-      Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+      this.rowBufferOnErrorContinue.setupSchema(Collections.singletonList(testCol));
       Assert.fail("Expected error");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.UNKNOWN_DATA_TYPE.getMessageCode(), e.getVendorCode());
@@ -179,7 +169,7 @@ public void buildFieldErrorStates() {
     testCol.setPhysicalType("SB1");
     testCol.setLogicalType("TIMESTAMP_TZ");
     try {
-      Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+      this.rowBufferOnErrorContinue.setupSchema(Collections.singletonList(testCol));
       Assert.fail("Expected error");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.UNKNOWN_DATA_TYPE.getMessageCode(), e.getVendorCode());
@@ -190,363 +180,13 @@ public void buildFieldErrorStates() {
     testCol.setPhysicalType("SB16");
     testCol.setLogicalType("TIME");
     try {
-      Field result = this.rowBufferOnErrorContinue.buildField(testCol);
+      this.rowBufferOnErrorContinue.setupSchema(Collections.singletonList(testCol));
       Assert.fail("Expected error");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.UNKNOWN_DATA_TYPE.getMessageCode(), e.getVendorCode());
     }
   }
 
-  @Test
-  public void buildFieldFixedSB1() {
-    // FIXED, SB1
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB1");
-    testCol.setNullable(true);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("testCol", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.TINYINT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB1");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "FIXED");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("nullable"), "true");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldFixedSB2() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB2");
-    testCol.setNullable(false);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("testCol", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.SMALLINT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB2");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "FIXED");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("nullable"), "false");
-    Assert.assertFalse(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldFixedSB4() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB4");
-    testCol.setNullable(true);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("testCol", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.INT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB4");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "FIXED");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldFixedSB8() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("testCol", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.BIGINT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "FIXED");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldFixedSB16() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB16");
-    testCol.setNullable(true);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    ArrowType expectedType =
-        new ArrowType.Decimal(testCol.getPrecision(), testCol.getScale(), DECIMAL_BIT_WIDTH);
-
-    Assert.assertEquals("testCol", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), expectedType);
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB16");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "FIXED");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldLobVariant() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("LOB");
-    testCol.setNullable(true);
-    testCol.setLogicalType("VARIANT");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("testCol", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.VARCHAR.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "LOB");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "VARIANT");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldTimestampNtzSB8() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIMESTAMP_NTZ");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("testCol", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.BIGINT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIMESTAMP_NTZ");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldTimestampNtzSB16() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB16");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIMESTAMP_NTZ");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("testCol", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.STRUCT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB16");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIMESTAMP_NTZ");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 2);
-    Assert.assertEquals(
-        result.getChildren().get(0).getFieldType().getType(), Types.MinorType.BIGINT.getType());
-    Assert.assertEquals(
-        result.getChildren().get(1).getFieldType().getType(), Types.MinorType.INT.getType());
-  }
-
-  @Test
-  public void buildFieldTimestampTzSB8() throws Exception {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIMESTAMP_TZ");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("testCol", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.STRUCT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIMESTAMP_TZ");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 2);
-    Assert.assertEquals(
-        result.getChildren().get(0).getFieldType().getType(), Types.MinorType.BIGINT.getType());
-    Assert.assertEquals(
-        result.getChildren().get(1).getFieldType().getType(), Types.MinorType.INT.getType());
-  }
-
-  @Test
-  public void buildFieldTimestampTzSB16() throws Exception {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB16");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIMESTAMP_TZ");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("testCol", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.STRUCT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB16");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIMESTAMP_TZ");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 3);
-    Assert.assertEquals(
-        result.getChildren().get(0).getFieldType().getType(), Types.MinorType.BIGINT.getType());
-    Assert.assertEquals(
-        result.getChildren().get(1).getFieldType().getType(), Types.MinorType.INT.getType());
-    Assert.assertEquals(
-        result.getChildren().get(2).getFieldType().getType(), Types.MinorType.INT.getType());
-  }
-
-  @Test
-  public void buildFieldTimestampDate() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("DATE");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("testCol", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.DATEDAY.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "DATE");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldTimeSB4() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB4");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIME");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("testCol", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.INT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB4");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIME");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldTimeSB8() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIME");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("testCol", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.BIGINT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIME");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldBoolean() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("BINARY");
-    testCol.setNullable(true);
-    testCol.setLogicalType("BOOLEAN");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("testCol", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.BIT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "BINARY");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "BOOLEAN");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldRealSB16() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB16");
-    testCol.setNullable(true);
-    testCol.setLogicalType("REAL");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("testCol", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.FLOAT8.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB16");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "REAL");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
   @Test
   public void testReset() {
     RowBufferStats stats = this.rowBufferOnErrorContinue.statsMap.get("COLCHAR");
@@ -630,7 +270,7 @@ public void testRowIndexWithMultipleRowsWithErrorr() {
     Assert.assertEquals(1, response.getInsertErrors().get(0).getRowIndex());
   }
 
-  private void testStringLengthHelper(ArrowRowBuffer rowBuffer) {
+  private void testStringLengthHelper(AbstractRowBuffer<?> rowBuffer) {
     Map<String, Object> row = new HashMap<>();
     row.put("colTinyInt", (byte) 1);
     row.put("\"colTinyInt\"", (byte) 1);
@@ -674,7 +314,7 @@ public void testInsertRow() {
     testInsertRowHelper(this.rowBufferOnErrorAbort);
   }
 
-  private void testInsertRowHelper(ArrowRowBuffer rowBuffer) {
+  private void testInsertRowHelper(AbstractRowBuffer<?> rowBuffer) {
     Map<String, Object> row = new HashMap<>();
     row.put("colTinyInt", (byte) 1);
     row.put("\"colTinyInt\"", (byte) 1);
@@ -694,7 +334,7 @@ public void testInsertRows() {
     testInsertRowsHelper(this.rowBufferOnErrorAbort);
   }
 
-  private void testInsertRowsHelper(ArrowRowBuffer rowBuffer) {
+  private void testInsertRowsHelper(AbstractRowBuffer<?> rowBuffer) {
     Map<String, Object> row1 = new HashMap<>();
     row1.put("colTinyInt", (byte) 1);
     row1.put("\"colTinyInt\"", (byte) 1);
@@ -742,7 +382,7 @@ public void testFlush() {
     testFlushHelper(this.rowBufferOnErrorContinue);
   }
 
-  private void testFlushHelper(ArrowRowBuffer rowBuffer) {
+  private void testFlushHelper(AbstractRowBuffer<?> rowBuffer) {
     String offsetToken = "1";
     Map<String, Object> row1 = new HashMap<>();
     row1.put("colTinyInt", (byte) 1);
@@ -767,23 +407,21 @@ private void testFlushHelper(ArrowRowBuffer rowBuffer) {
     Assert.assertFalse(response.hasErrors());
     float bufferSize = rowBuffer.getSize();
 
-    ChannelData<VectorSchemaRoot> data = rowBuffer.flush();
+    ChannelData<?> data = rowBuffer.flush();
     Assert.assertEquals(2, data.getRowCount());
     Assert.assertEquals((Long) 1L, data.getRowSequencer());
-    Assert.assertEquals(7, data.getVectors().getFieldVectors().size());
     Assert.assertEquals(offsetToken, data.getOffsetToken());
     Assert.assertEquals(bufferSize, data.getBufferSize(), 0);
   }
 
   @Test
   public void testDoubleQuotesColumnName() {
-    testDoubleQuotesColumnNameHelper(this.channelOnErrorAbort);
-    testDoubleQuotesColumnNameHelper(this.channelOnErrorContinue);
+    testDoubleQuotesColumnNameHelper(OpenChannelRequest.OnErrorOption.ABORT);
+    testDoubleQuotesColumnNameHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
   }
 
-  private void testDoubleQuotesColumnNameHelper(
-      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
-    ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
+  private void testDoubleQuotesColumnNameHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
+    AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colDoubleQuotes = new ColumnMetadata();
     colDoubleQuotes.setName("\"colDoubleQuotes\"");
@@ -873,12 +511,12 @@ public void testBuildEpInfoFromNullColumnStats() {
   }
 
   @Test
-  public void testArrowE2E() {
-    testArrowE2EHelper(this.rowBufferOnErrorAbort);
-    testArrowE2EHelper(this.rowBufferOnErrorContinue);
+  public void testE2E() {
+    testE2EHelper(this.rowBufferOnErrorAbort);
+    testE2EHelper(this.rowBufferOnErrorContinue);
   }
 
-  private void testArrowE2EHelper(ArrowRowBuffer rowBuffer) {
+  private void testE2EHelper(AbstractRowBuffer<?> rowBuffer) {
     Map<String, Object> row1 = new HashMap<>();
     row1.put("\"colTinyInt\"", (byte) 10);
     row1.put("colTinyInt", (byte) 1);
@@ -891,216 +529,22 @@ private void testArrowE2EHelper(ArrowRowBuffer rowBuffer) {
     InsertValidationResponse response = rowBuffer.insertRows(Collections.singletonList(row1), null);
     Assert.assertFalse(response.hasErrors());
 
-    Assert.assertEquals((byte) 10, rowBuffer.vectorsRoot.getVector("\"colTinyInt\"").getObject(0));
-    Assert.assertEquals((byte) 1, rowBuffer.vectorsRoot.getVector("COLTINYINT").getObject(0));
-    Assert.assertEquals((short) 2, rowBuffer.vectorsRoot.getVector("COLSMALLINT").getObject(0));
-    Assert.assertEquals(3, rowBuffer.vectorsRoot.getVector("COLINT").getObject(0));
-    Assert.assertEquals(4L, rowBuffer.vectorsRoot.getVector("COLBIGINT").getObject(0));
-    Assert.assertEquals(
-        new BigDecimal("4.00"), rowBuffer.vectorsRoot.getVector("COLDECIMAL").getObject(0));
-    Assert.assertEquals(new Text("2"), rowBuffer.vectorsRoot.getVector("COLCHAR").getObject(0));
+    Assert.assertEquals((byte) 10, rowBuffer.getVectorValueAt("\"colTinyInt\"", 0));
+    Assert.assertEquals((byte) 1, rowBuffer.getVectorValueAt("COLTINYINT", 0));
+    Assert.assertEquals((short) 2, rowBuffer.getVectorValueAt("COLSMALLINT", 0));
+    Assert.assertEquals(3, rowBuffer.getVectorValueAt("COLINT", 0));
+    Assert.assertEquals(4L, rowBuffer.getVectorValueAt("COLBIGINT", 0));
+    Assert.assertEquals(new BigDecimal("4.00"), rowBuffer.getVectorValueAt("COLDECIMAL", 0));
+    Assert.assertEquals("2", rowBuffer.getVectorValueAt("COLCHAR", 0));
   }
 
   @Test
-  public void testArrowE2ETimestampLTZ() {
-    testArrowE2ETimestampLTZHelper(this.channelOnErrorContinue);
-    testArrowE2ETimestampLTZHelper(this.channelOnErrorAbort);
+  public void testE2ETimestampErrors() {
+    testE2ETimestampErrorsHelper(this.rowBufferOnErrorAbort);
+    testE2ETimestampErrorsHelper(this.rowBufferOnErrorContinue);
   }
 
-  private void testArrowE2ETimestampLTZHelper(
-      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
-    ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
-
-    ColumnMetadata colTimestampLtzSB8 = new ColumnMetadata();
-    colTimestampLtzSB8.setName("COLTIMESTAMPLTZ_SB8");
-    colTimestampLtzSB8.setPhysicalType("SB8");
-    colTimestampLtzSB8.setNullable(false);
-    colTimestampLtzSB8.setLogicalType("TIMESTAMP_LTZ");
-    colTimestampLtzSB8.setScale(0);
-
-    ColumnMetadata colTimestampLtzSB16 = new ColumnMetadata();
-    colTimestampLtzSB16.setName("COLTIMESTAMPLTZ_SB16");
-    colTimestampLtzSB16.setPhysicalType("SB16");
-    colTimestampLtzSB16.setNullable(false);
-    colTimestampLtzSB16.setLogicalType("TIMESTAMP_LTZ");
-    colTimestampLtzSB16.setScale(9);
-
-    innerBuffer.setupSchema(Arrays.asList(colTimestampLtzSB8, colTimestampLtzSB16));
-
-    Map<String, Object> row = new HashMap<>();
-    row.put("COLTIMESTAMPLTZ_SB8", "1621899220");
-    row.put("COLTIMESTAMPLTZ_SB16", new BigDecimal("1621899220.123456789"));
-
-    InsertValidationResponse response =
-        innerBuffer.insertRows(Collections.singletonList(row), null);
-    Assert.assertFalse(response.hasErrors());
-    Assert.assertEquals(
-        1621899220l, innerBuffer.vectorsRoot.getVector("COLTIMESTAMPLTZ_SB8").getObject(0));
-    Assert.assertEquals(
-        "epoch",
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPLTZ_SB16")
-            .getChildrenFromFields()
-            .get(0)
-            .getName());
-    Assert.assertEquals(
-        1621899220l,
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPLTZ_SB16")
-            .getChildrenFromFields()
-            .get(0)
-            .getObject(0));
-    Assert.assertEquals(
-        "fraction",
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPLTZ_SB16")
-            .getChildrenFromFields()
-            .get(1)
-            .getName());
-    Assert.assertEquals(
-        123456789,
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPLTZ_SB16")
-            .getChildrenFromFields()
-            .get(1)
-            .getObject(0));
-  }
-
-  @Test
-  public void testArrowE2ETimestampTZ() {
-    testArrowE2ETimestampTZHelper(this.channelOnErrorContinue);
-    testArrowE2ETimestampTZHelper(this.channelOnErrorAbort);
-  }
-
-  private void testArrowE2ETimestampTZHelper(
-      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
-    ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
-
-    ColumnMetadata colTimestampTzSB8 = new ColumnMetadata();
-    colTimestampTzSB8.setName("COLTIMESTAMPTZ_SB8");
-    colTimestampTzSB8.setPhysicalType("SB8");
-    colTimestampTzSB8.setNullable(false);
-    colTimestampTzSB8.setLogicalType("TIMESTAMP_TZ");
-    colTimestampTzSB8.setScale(0);
-
-    ColumnMetadata colTimestampTzSB16 = new ColumnMetadata();
-    colTimestampTzSB16.setName("COLTIMESTAMPTZ_SB16");
-    colTimestampTzSB16.setPhysicalType("SB16");
-    colTimestampTzSB16.setNullable(false);
-    colTimestampTzSB16.setLogicalType("TIMESTAMP_TZ");
-    colTimestampTzSB16.setScale(9);
-
-    innerBuffer.setupSchema(Arrays.asList(colTimestampTzSB8, colTimestampTzSB16));
-
-    Map<String, Object> row = new HashMap<>();
-    row.put("COLTIMESTAMPTZ_SB8", "2021-01-01 01:00:00 +0100");
-    row.put("COLTIMESTAMPTZ_SB16", "2021-01-01 10:00:00.123456789 +1000");
-
-    InsertValidationResponse response =
-        innerBuffer.insertRows(Collections.singletonList(row), null);
-    Assert.assertFalse(response.hasErrors());
-
-    // SB8
-    Assert.assertEquals(
-        "epoch",
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPTZ_SB8")
-            .getChildrenFromFields()
-            .get(0)
-            .getName());
-    Assert.assertEquals(
-        1609459200l,
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPTZ_SB8")
-            .getChildrenFromFields()
-            .get(0)
-            .getObject(0));
-
-    Assert.assertEquals(
-        "timezone",
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPTZ_SB8")
-            .getChildrenFromFields()
-            .get(1)
-            .getName());
-    Assert.assertEquals(
-        1500,
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPTZ_SB8")
-            .getChildrenFromFields()
-            .get(1)
-            .getObject(0));
-
-    // SB16
-    Assert.assertEquals(
-        "epoch",
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPTZ_SB16")
-            .getChildrenFromFields()
-            .get(0)
-            .getName());
-    Assert.assertEquals(
-        1609459200l,
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPTZ_SB16")
-            .getChildrenFromFields()
-            .get(0)
-            .getObject(0));
-
-    Assert.assertEquals(
-        "fraction",
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPTZ_SB16")
-            .getChildrenFromFields()
-            .get(1)
-            .getName());
-    Assert.assertEquals(
-        123456789,
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPTZ_SB16")
-            .getChildrenFromFields()
-            .get(1)
-            .getObject(0));
-
-    Assert.assertEquals(
-        "timezone",
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPTZ_SB16")
-            .getChildrenFromFields()
-            .get(2)
-            .getName());
-    Assert.assertEquals(
-        2040,
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPTZ_SB16")
-            .getChildrenFromFields()
-            .get(2)
-            .getObject(0));
-  }
-
-  @Test
-  public void testArrowE2ETimestampErrors() {
-    testArrowE2ETimestampErrorsHelper(this.channelOnErrorAbort);
-    testArrowE2ETimestampErrorsHelper(this.channelOnErrorContinue);
-  }
-
-  private void testArrowE2ETimestampErrorsHelper(
-      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
-    ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
+  private void testE2ETimestampErrorsHelper(AbstractRowBuffer<?> innerBuffer) {
 
     ColumnMetadata colTimestampLtzSB16 = new ColumnMetadata();
     colTimestampLtzSB16.setName("COLTIMESTAMPLTZ_SB16");
@@ -1115,7 +559,7 @@ private void testArrowE2ETimestampErrorsHelper(
     row.put("COLTIMESTAMPLTZ_SB8", "1621899220");
     row.put("COLTIMESTAMPLTZ_SB16", "1621899220.1234567");
 
-    if (channel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.CONTINUE) {
+    if (innerBuffer.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.CONTINUE) {
       InsertValidationResponse response =
           innerBuffer.insertRows(Collections.singletonList(row), null);
       Assert.assertTrue(response.hasErrors());
@@ -1124,8 +568,7 @@ private void testArrowE2ETimestampErrorsHelper(
           response.getInsertErrors().get(0).getException().getVendorCode());
     } else {
       try {
-        InsertValidationResponse response =
-            innerBuffer.insertRows(Collections.singletonList(row), null);
+        innerBuffer.insertRows(Collections.singletonList(row), null);
       } catch (SFException e) {
         Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
       }
@@ -1138,7 +581,7 @@ public void testStatsE2E() {
     testStatsE2EHelper(this.rowBufferOnErrorContinue);
   }
 
-  private void testStatsE2EHelper(ArrowRowBuffer rowBuffer) {
+  private void testStatsE2EHelper(AbstractRowBuffer<?> rowBuffer) {
     Map<String, Object> row1 = new HashMap<>();
     row1.put("\"colTinyInt\"", (byte) 10);
     row1.put("colTinyInt", (byte) 1);
@@ -1159,7 +602,7 @@ private void testStatsE2EHelper(ArrowRowBuffer rowBuffer) {
 
     InsertValidationResponse response = rowBuffer.insertRows(Arrays.asList(row1, row2), null);
     Assert.assertFalse(response.hasErrors());
-    ChannelData<VectorSchemaRoot> result = rowBuffer.flush();
+    ChannelData<?> result = rowBuffer.flush();
     Map<String, RowBufferStats> columnEpStats = result.getColumnEps();
 
     Assert.assertEquals(
@@ -1201,19 +644,18 @@ private void testStatsE2EHelper(ArrowRowBuffer rowBuffer) {
     Assert.assertEquals(-1, columnEpStats.get("COLCHAR").getDistinctValues());
 
     // Confirm we reset
-    ChannelData<VectorSchemaRoot> resetResults = rowBuffer.flush();
+    ChannelData<?> resetResults = rowBuffer.flush();
     Assert.assertNull(resetResults);
   }
 
   @Test
   public void testStatsE2ETimestamp() {
-    testStatsE2ETimestampHelper(this.channelOnErrorAbort);
-    testStatsE2ETimestampHelper(this.channelOnErrorContinue);
+    testStatsE2ETimestampHelper(OpenChannelRequest.OnErrorOption.ABORT);
+    testStatsE2ETimestampHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
   }
 
-  private void testStatsE2ETimestampHelper(
-      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
-    ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
+  private void testStatsE2ETimestampHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
+    AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colTimestampLtzSB8 = new ColumnMetadata();
     colTimestampLtzSB8.setName("COLTIMESTAMPLTZ_SB8");
@@ -1257,7 +699,7 @@ private void testStatsE2ETimestampHelper(
     InsertValidationResponse response =
         innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null);
     Assert.assertFalse(response.hasErrors());
-    ChannelData<VectorSchemaRoot> result = innerBuffer.flush();
+    ChannelData<?> result = innerBuffer.flush();
     Assert.assertEquals(3, result.getRowCount());
 
     Assert.assertEquals(
@@ -1289,13 +731,12 @@ private void testStatsE2ETimestampHelper(
 
   @Test
   public void testE2EDate() {
-    testE2EDateHelper(this.channelOnErrorContinue);
-    testE2EDateHelper(this.channelOnErrorAbort);
+    testE2EDateHelper(OpenChannelRequest.OnErrorOption.ABORT);
+    testE2EDateHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
   }
 
-  private void testE2EDateHelper(
-      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
-    ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
+  private void testE2EDateHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
+    AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colDate = new ColumnMetadata();
     colDate.setName("COLDATE");
@@ -1319,13 +760,13 @@ private void testE2EDateHelper(
         innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null);
     Assert.assertFalse(response.hasErrors());
 
-    // Check data was inserted into Arrow correctly
-    Assert.assertEquals(18772, innerBuffer.vectorsRoot.getVector("COLDATE").getObject(0));
-    Assert.assertEquals(18773, innerBuffer.vectorsRoot.getVector("COLDATE").getObject(1));
-    Assert.assertNull(innerBuffer.vectorsRoot.getVector("COLDATE").getObject(2));
+    // Check data was inserted into the buffer correctly
+    Assert.assertEquals(18772, innerBuffer.getVectorValueAt("COLDATE", 0));
+    Assert.assertEquals(18773, innerBuffer.getVectorValueAt("COLDATE", 1));
+    Assert.assertNull(innerBuffer.getVectorValueAt("COLDATE", 2));
 
     // Check stats generation
-    ChannelData<VectorSchemaRoot> result = innerBuffer.flush();
+    ChannelData<?> result = innerBuffer.flush();
     Assert.assertEquals(3, result.getRowCount());
 
     Assert.assertEquals(
@@ -1338,13 +779,12 @@ private void testE2EDateHelper(
 
   @Test
   public void testE2ETime() {
-    testE2ETimeHelper(this.channelOnErrorAbort);
-    testE2ETimeHelper(this.channelOnErrorContinue);
+    testE2ETimeHelper(OpenChannelRequest.OnErrorOption.ABORT);
+    testE2ETimeHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
   }
 
-  private void testE2ETimeHelper(
-      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
-    ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
+  private void testE2ETimeHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
+    AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colTimeSB4 = new ColumnMetadata();
     colTimeSB4.setName("COLTIMESB4");
@@ -1378,17 +818,17 @@ private void testE2ETimeHelper(
         innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null);
     Assert.assertFalse(response.hasErrors());
 
-    // Check data was inserted into Arrow correctly
-    Assert.assertEquals(43200, innerBuffer.vectorsRoot.getVector("COLTIMESB4").getObject(0));
-    Assert.assertEquals(43260, innerBuffer.vectorsRoot.getVector("COLTIMESB4").getObject(1));
-    Assert.assertNull(innerBuffer.vectorsRoot.getVector("COLTIMESB4").getObject(2));
+    // Check data was inserted into the buffer correctly
+    Assert.assertEquals(43200, innerBuffer.getVectorValueAt("COLTIMESB4", 0));
+    Assert.assertEquals(43260, innerBuffer.getVectorValueAt("COLTIMESB4", 1));
+    Assert.assertNull(innerBuffer.getVectorValueAt("COLTIMESB4", 2));
 
-    Assert.assertEquals(44200123l, innerBuffer.vectorsRoot.getVector("COLTIMESB8").getObject(0));
-    Assert.assertEquals(44201000l, innerBuffer.vectorsRoot.getVector("COLTIMESB8").getObject(1));
-    Assert.assertNull(innerBuffer.vectorsRoot.getVector("COLTIMESB8").getObject(2));
+    Assert.assertEquals(44200123L, innerBuffer.getVectorValueAt("COLTIMESB8", 0));
+    Assert.assertEquals(44201000L, innerBuffer.getVectorValueAt("COLTIMESB8", 1));
+    Assert.assertNull(innerBuffer.getVectorValueAt("COLTIMESB8", 2));
 
     // Check stats generation
-    ChannelData<VectorSchemaRoot> result = innerBuffer.flush();
+    ChannelData<?> result = innerBuffer.flush();
     Assert.assertEquals(3, result.getRowCount());
 
     Assert.assertEquals(
@@ -1408,13 +848,12 @@ private void testE2ETimeHelper(
 
   @Test
   public void testNullableCheck() {
-    testNullableCheckHelper(this.channelOnErrorContinue);
-    testNullableCheckHelper(this.channelOnErrorAbort);
+    testNullableCheckHelper(OpenChannelRequest.OnErrorOption.ABORT);
+    testNullableCheckHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
   }
 
-  private void testNullableCheckHelper(
-      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
-    ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
+  private void testNullableCheckHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
+    AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colBoolean = new ColumnMetadata();
     colBoolean.setName("COLBOOLEAN");
@@ -1429,10 +868,9 @@ private void testNullableCheckHelper(
 
     InsertValidationResponse response = innerBuffer.insertRows(Collections.singletonList(row), "1");
     Assert.assertFalse(response.hasErrors());
-    ;
 
     row.put("COLBOOLEAN", null);
-    if (channel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.CONTINUE) {
+    if (innerBuffer.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.CONTINUE) {
       response = innerBuffer.insertRows(Collections.singletonList(row), "1");
       Assert.assertTrue(response.hasErrors());
       Assert.assertEquals(
@@ -1449,13 +887,12 @@ private void testNullableCheckHelper(
 
   @Test
   public void testMissingColumnCheck() {
-    testMissingColumnCheckHelper(this.channelOnErrorContinue);
-    testMissingColumnCheckHelper(this.channelOnErrorAbort);
+    testMissingColumnCheckHelper(OpenChannelRequest.OnErrorOption.ABORT);
+    testMissingColumnCheckHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
   }
 
-  private void testMissingColumnCheckHelper(
-      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
-    ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
+  private void testMissingColumnCheckHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
+    AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colBoolean = new ColumnMetadata();
     colBoolean.setName("COLBOOLEAN");
@@ -1480,7 +917,7 @@ private void testMissingColumnCheckHelper(
 
     Map<String, Object> row2 = new HashMap<>();
     row2.put("COLBOOLEAN2", true);
-    if (channel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.CONTINUE) {
+    if (innerBuffer.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.CONTINUE) {
       response = innerBuffer.insertRows(Collections.singletonList(row2), "2");
       Assert.assertTrue(response.hasErrors());
       InsertValidationResponse.InsertError error = response.getInsertErrors().get(0);
@@ -1499,7 +936,7 @@ private void testMissingColumnCheckHelper(
 
   @Test
   public void testExtraColumnsCheck() {
-    ArrowRowBuffer innerBuffer = new ArrowRowBuffer(this.channelOnErrorContinue);
+    AbstractRowBuffer<?> innerBuffer = createTestBuffer(OpenChannelRequest.OnErrorOption.CONTINUE);
 
     ColumnMetadata colBoolean = new ColumnMetadata();
     colBoolean.setName("COLBOOLEAN1");
@@ -1524,13 +961,12 @@ public void testExtraColumnsCheck() {
 
   @Test
   public void testE2EBoolean() {
-    testE2EBooleanHelper(this.channelOnErrorContinue);
-    testE2EBooleanHelper(this.channelOnErrorAbort);
+    testE2EBooleanHelper(OpenChannelRequest.OnErrorOption.ABORT);
+    testE2EBooleanHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
   }
 
-  private void testE2EBooleanHelper(
-      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
-    ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
+  private void testE2EBooleanHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
+    AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colBoolean = new ColumnMetadata();
     colBoolean.setName("COLBOOLEAN");
@@ -1555,13 +991,13 @@ private void testE2EBooleanHelper(
         innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null);
     Assert.assertFalse(response.hasErrors());
 
-    // Check data was inserted into Arrow correctly
-    Assert.assertEquals(true, innerBuffer.vectorsRoot.getVector("COLBOOLEAN").getObject(0));
-    Assert.assertEquals(false, innerBuffer.vectorsRoot.getVector("COLBOOLEAN").getObject(1));
-    Assert.assertNull(innerBuffer.vectorsRoot.getVector("COLBOOLEAN").getObject(2));
+    // Check data was inserted into the buffer correctly
+    Assert.assertEquals(true, innerBuffer.getVectorValueAt("COLBOOLEAN", 0));
+    Assert.assertEquals(false, innerBuffer.getVectorValueAt("COLBOOLEAN", 1));
+    Assert.assertNull(innerBuffer.getVectorValueAt("COLBOOLEAN", 2));
 
     // Check stats generation
-    ChannelData<VectorSchemaRoot> result = innerBuffer.flush();
+    ChannelData<?> result = innerBuffer.flush();
     Assert.assertEquals(3, result.getRowCount());
 
     Assert.assertEquals(
@@ -1573,13 +1009,12 @@ private void testE2EBooleanHelper(
 
   @Test
   public void testE2EBinary() {
-    testE2EBinaryHelper(this.channelOnErrorAbort);
-    testE2EBinaryHelper(this.channelOnErrorContinue);
+    testE2EBinaryHelper(OpenChannelRequest.OnErrorOption.ABORT);
+    testE2EBinaryHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
   }
 
-  private void testE2EBinaryHelper(
-      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
-    ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
+  private void testE2EBinaryHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
+    AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colBinary = new ColumnMetadata();
     colBinary.setName("COLBINARY");
@@ -1603,21 +1038,17 @@ private void testE2EBinaryHelper(
         innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null);
     Assert.assertFalse(response.hasErrors());
 
-    // Check data was inserted into Arrow correctly
+    // Check data was inserted into the buffer correctly
     Assert.assertEquals(
         "Hello World",
-        new String(
-            (byte[]) innerBuffer.vectorsRoot.getVector("COLBINARY").getObject(0),
-            StandardCharsets.UTF_8));
+        new String((byte[]) innerBuffer.getVectorValueAt("COLBINARY", 0), StandardCharsets.UTF_8));
     Assert.assertEquals(
         "Honk Honk",
-        new String(
-            (byte[]) innerBuffer.vectorsRoot.getVector("COLBINARY").getObject(1),
-            StandardCharsets.UTF_8));
-    Assert.assertNull(innerBuffer.vectorsRoot.getVector("COLBINARY").getObject(2));
+        new String((byte[]) innerBuffer.getVectorValueAt("COLBINARY", 1), StandardCharsets.UTF_8));
+    Assert.assertNull(innerBuffer.getVectorValueAt("COLBINARY", 2));
 
     // Check stats generation
-    ChannelData<VectorSchemaRoot> result = innerBuffer.flush();
+    ChannelData<?> result = innerBuffer.flush();
 
     Assert.assertEquals(3, result.getRowCount());
     Assert.assertEquals(11L, result.getColumnEps().get("COLBINARY").getCurrentMaxLength());
@@ -1630,13 +1061,12 @@ private void testE2EBinaryHelper(
 
   @Test
   public void testE2EReal() {
-    testE2ERealHelper(this.channelOnErrorContinue);
-    testE2ERealHelper(this.channelOnErrorAbort);
+    testE2ERealHelper(OpenChannelRequest.OnErrorOption.ABORT);
+    testE2ERealHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
   }
 
-  private void testE2ERealHelper(
-      SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
-    ArrowRowBuffer innerBuffer = new ArrowRowBuffer(channel);
+  private void testE2ERealHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
+    AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colReal = new ColumnMetadata();
     colReal.setName("COLREAL");
@@ -1660,13 +1090,13 @@ private void testE2ERealHelper(
         innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null);
     Assert.assertFalse(response.hasErrors());
 
-    // Check data was inserted into Arrow correctly
-    Assert.assertEquals(123.456, innerBuffer.vectorsRoot.getVector("COLREAL").getObject(0));
-    Assert.assertEquals(123.4567, innerBuffer.vectorsRoot.getVector("COLREAL").getObject(1));
-    Assert.assertNull(innerBuffer.vectorsRoot.getVector("COLREAL").getObject(2));
+    // Check data was inserted into the buffer correctly
+    Assert.assertEquals(123.456, innerBuffer.getVectorValueAt("COLREAL", 0));
+    Assert.assertEquals(123.4567, innerBuffer.getVectorValueAt("COLREAL", 1));
+    Assert.assertNull(innerBuffer.getVectorValueAt("COLREAL", 2));
 
     // Check stats generation
-    ChannelData<VectorSchemaRoot> result = innerBuffer.flush();
+    ChannelData<?> result = innerBuffer.flush();
 
     Assert.assertEquals(3, result.getRowCount());
     Assert.assertEquals(
@@ -1678,7 +1108,7 @@ private void testE2ERealHelper(
 
   @Test
   public void testOnErrorAbortFailures() {
-    ArrowRowBuffer innerBuffer = new ArrowRowBuffer(this.channelOnErrorAbort);
+    AbstractRowBuffer<?> innerBuffer = createTestBuffer(OpenChannelRequest.OnErrorOption.ABORT);
 
     ColumnMetadata colDecimal = new ColumnMetadata();
     colDecimal.setName("COLDECIMAL");
@@ -1696,7 +1126,7 @@ public void testOnErrorAbortFailures() {
     Assert.assertFalse(response.hasErrors());
 
     Assert.assertEquals(1, innerBuffer.rowCount);
-    Assert.assertEquals(0, innerBuffer.tempVectorsRoot.getRowCount());
+    Assert.assertEquals(0, innerBuffer.getTempRowCount());
     Assert.assertEquals(
         1, innerBuffer.statsMap.get("COLDECIMAL").getCurrentMaxIntValue().intValue());
     Assert.assertEquals(
@@ -1710,7 +1140,7 @@ public void testOnErrorAbortFailures() {
     Assert.assertFalse(response.hasErrors());
 
     Assert.assertEquals(2, innerBuffer.rowCount);
-    Assert.assertEquals(0, innerBuffer.tempVectorsRoot.getRowCount());
+    Assert.assertEquals(0, innerBuffer.getTempRowCount());
     Assert.assertEquals(
         2, innerBuffer.statsMap.get("COLDECIMAL").getCurrentMaxIntValue().intValue());
     Assert.assertEquals(
@@ -1727,7 +1157,7 @@ public void testOnErrorAbortFailures() {
     }
 
     Assert.assertEquals(2, innerBuffer.rowCount);
-    Assert.assertEquals(0, innerBuffer.tempVectorsRoot.getRowCount());
+    Assert.assertEquals(0, innerBuffer.getTempRowCount());
     Assert.assertEquals(
         2, innerBuffer.statsMap.get("COLDECIMAL").getCurrentMaxIntValue().intValue());
     Assert.assertEquals(
@@ -1739,7 +1169,7 @@ public void testOnErrorAbortFailures() {
     response = innerBuffer.insertRows(Collections.singletonList(row3), "3");
     Assert.assertFalse(response.hasErrors());
     Assert.assertEquals(3, innerBuffer.rowCount);
-    Assert.assertEquals(0, innerBuffer.tempVectorsRoot.getRowCount());
+    Assert.assertEquals(0, innerBuffer.getTempRowCount());
     Assert.assertEquals(
         3, innerBuffer.statsMap.get("COLDECIMAL").getCurrentMaxIntValue().intValue());
     Assert.assertEquals(
@@ -1747,7 +1177,7 @@ public void testOnErrorAbortFailures() {
     Assert.assertNull(innerBuffer.tempStatsMap.get("COLDECIMAL").getCurrentMaxIntValue());
     Assert.assertNull(innerBuffer.tempStatsMap.get("COLDECIMAL").getCurrentMinIntValue());
 
-    ChannelData<VectorSchemaRoot> data = innerBuffer.flush();
+    ChannelData<?> data = innerBuffer.flush();
     Assert.assertEquals(3, data.getRowCount());
     Assert.assertEquals(0, innerBuffer.rowCount);
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 82ca97f94..a7710932d 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -49,9 +49,9 @@ public void testChannelFactoryNullFields() {
     String dbName = "DATABASE";
     String schemaName = "SCHEMA";
     String tableName = "TABLE";
-    Long channelSequencer = 0L;
-    Long rowSequencer = 0L;
-    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
+    long channelSequencer = 0L;
+    long rowSequencer = 0L;
+    SnowflakeStreamingIngestClientInternal<StubChunkData> client =
         new SnowflakeStreamingIngestClientInternal<>("client");
 
     Object[] fields =
@@ -61,15 +61,14 @@ public void testChannelFactoryNullFields() {
       Object tmp = fields[i];
       fields[i] = null;
       try {
-        SnowflakeStreamingIngestChannelInternal channel =
-            SnowflakeStreamingIngestChannelFactory.builder((String) fields[0])
-                .setDBName((String) fields[1])
-                .setSchemaName((String) fields[2])
-                .setTableName((String) fields[3])
-                .setRowSequencer((Long) fields[4])
-                .setChannelSequencer((Long) fields[5])
-                .setOwningClient((SnowflakeStreamingIngestClientInternal) fields[6])
-                .build();
+        SnowflakeStreamingIngestChannelFactory.<StubChunkData>builder((String) fields[0])
+            .setDBName((String) fields[1])
+            .setSchemaName((String) fields[2])
+            .setTableName((String) fields[3])
+            .setRowSequencer((Long) fields[4])
+            .setChannelSequencer((Long) fields[5])
+            .setOwningClient((SnowflakeStreamingIngestClientInternal<StubChunkData>) fields[6])
+            .build();
         Assert.fail("Channel factory should fail with null fields");
       } catch (SFException e) {
         Assert.assertTrue(
@@ -89,13 +88,13 @@ public void testChannelFactorySuccess() {
     String offsetToken = "0";
     String encryptionKey = "key";
     Long channelSequencer = 0L;
-    Long rowSequencer = 0L;
+    long rowSequencer = 0L;
 
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal("client");
+    SnowflakeStreamingIngestClientInternal<StubChunkData> client =
+        new SnowflakeStreamingIngestClientInternal<>("client");
 
-    SnowflakeStreamingIngestChannelInternal channel =
-        SnowflakeStreamingIngestChannelFactory.builder(name)
+    SnowflakeStreamingIngestChannelInternal<StubChunkData> channel =
+        SnowflakeStreamingIngestChannelFactory.<StubChunkData>builder(name)
             .setDBName(dbName)
             .setSchemaName(schemaName)
             .setTableName(tableName)
@@ -125,10 +124,10 @@ public void testChannelFactorySuccess() {
 
   @Test
   public void testChannelValid() {
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal("client");
-    SnowflakeStreamingIngestChannelInternal channel =
-        new SnowflakeStreamingIngestChannelInternal(
+    SnowflakeStreamingIngestClientInternal<StubChunkData> client =
+        new SnowflakeStreamingIngestClientInternal<>("client");
+    SnowflakeStreamingIngestChannelInternal<StubChunkData> channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel",
             "db",
             "schema",
@@ -139,12 +138,11 @@ public void testChannelValid() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
 
     Assert.assertTrue(channel.isValid());
     channel.invalidate();
-    Assert.assertTrue(!channel.isValid());
+    Assert.assertFalse(channel.isValid());
 
     // Can't insert rows to invalid channel
     try {
@@ -175,10 +173,10 @@ public void testChannelValid() {
 
   @Test
   public void testChannelClose() {
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal("client");
-    SnowflakeStreamingIngestChannelInternal channel =
-        new SnowflakeStreamingIngestChannelInternal(
+    SnowflakeStreamingIngestClientInternal<StubChunkData> client =
+        new SnowflakeStreamingIngestClientInternal<>("client");
+    SnowflakeStreamingIngestChannelInternal<StubChunkData> channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel",
             "db",
             "schema",
@@ -189,10 +187,9 @@ public void testChannelClose() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
 
-    Assert.assertTrue(!channel.isClosed());
+    Assert.assertFalse(channel.isClosed());
     channel.markClosed();
     Assert.assertTrue(channel.isClosed());
 
@@ -210,11 +207,10 @@ public void testChannelClose() {
   @Test
   public void testOpenChannelRequestCreationMissingField() {
     try {
-      OpenChannelRequest request =
-          OpenChannelRequest.builder("CHANNEL")
-              .setDBName("STREAMINGINGEST_TEST")
-              .setSchemaName("PUBLIC")
-              .build();
+      OpenChannelRequest.builder("CHANNEL")
+          .setDBName("STREAMINGINGEST_TEST")
+          .setSchemaName("PUBLIC")
+          .build();
       Assert.fail("Request should fail with missing table name");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.NULL_OR_EMPTY_STRING.getMessageCode(), e.getVendorCode());
@@ -263,8 +259,7 @@ public void testOpenChannelPostRequest() throws Exception {
             payload, OPEN_CHANNEL_ENDPOINT, "open channel");
 
     Assert.assertEquals(
-        String.format("%s%s", urlStr, OPEN_CHANNEL_ENDPOINT),
-        request.getRequestLine().getUri().toString());
+        String.format("%s%s", urlStr, OPEN_CHANNEL_ENDPOINT), request.getRequestLine().getUri());
     Assert.assertNotNull(request.getFirstHeader(HttpHeaders.USER_AGENT));
     Assert.assertNotNull(request.getFirstHeader(HttpHeaders.AUTHORIZATION));
     Assert.assertEquals("POST", request.getMethod());
@@ -291,8 +286,8 @@ public void testOpenChannelErrorResponse() throws Exception {
 
     RequestBuilder requestBuilder =
         new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair());
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal(
+    SnowflakeStreamingIngestClientInternal<StubChunkData> client =
+        new SnowflakeStreamingIngestClientInternal<>(
             "client",
             new SnowflakeURL("snowflake.dev.local:8082"),
             null,
@@ -310,7 +305,7 @@ public void testOpenChannelErrorResponse() throws Exception {
             .build();
 
     try {
-      SnowflakeStreamingIngestChannel channel = client.openChannel(request);
+      client.openChannel(request);
       Assert.fail("Open channel should fail on 500 error");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.OPEN_CHANNEL_FAILURE.getMessageCode(), e.getVendorCode());
@@ -361,8 +356,8 @@ public void testOpenChannelSnowflakeInternalErrorResponse() throws Exception {
 
     RequestBuilder requestBuilder =
         new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair());
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal(
+    SnowflakeStreamingIngestClientInternal<StubChunkData> client =
+        new SnowflakeStreamingIngestClientInternal<>(
             "client",
             new SnowflakeURL("snowflake.dev.local:8082"),
             null,
@@ -380,7 +375,7 @@ public void testOpenChannelSnowflakeInternalErrorResponse() throws Exception {
             .build();
 
     try {
-      SnowflakeStreamingIngestChannel channel = client.openChannel(request);
+      client.openChannel(request);
       Assert.fail("Open channel should fail on Snowflake internal error");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.OPEN_CHANNEL_FAILURE.getMessageCode(), e.getVendorCode());
@@ -441,8 +436,8 @@ public void testOpenChannelSuccessResponse() throws Exception {
 
     RequestBuilder requestBuilder =
         new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair());
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal(
+    SnowflakeStreamingIngestClientInternal<?> client =
+        new SnowflakeStreamingIngestClientInternal<>(
             "client",
             new SnowflakeURL("snowflake.dev.local:8082"),
             null,
@@ -468,10 +463,10 @@ public void testOpenChannelSuccessResponse() throws Exception {
 
   @Test
   public void testInsertRow() {
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal("client");
-    SnowflakeStreamingIngestChannelInternal channel =
-        new SnowflakeStreamingIngestChannelInternal(
+    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
+        new SnowflakeStreamingIngestClientInternal<>("client");
+    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel",
             "db",
             "schema",
@@ -482,8 +477,7 @@ public void testInsertRow() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
 
     ColumnMetadata col = new ColumnMetadata();
     col.setName("COL");
@@ -519,10 +513,10 @@ public void testInsertRow() {
 
   @Test
   public void testInsertRowThrottling() {
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal("client");
-    SnowflakeStreamingIngestChannelInternal channel =
-        new SnowflakeStreamingIngestChannelInternal(
+    SnowflakeStreamingIngestClientInternal<?> client =
+        new SnowflakeStreamingIngestClientInternal<>("client");
+    SnowflakeStreamingIngestChannelInternal<?> channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel",
             "db",
             "schema",
@@ -533,8 +527,7 @@ public void testInsertRowThrottling() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
 
     Runtime mockedRunTime = Mockito.mock(Runtime.class);
     Mockito.when(mockedRunTime.totalMemory()).thenReturn(1000000L);
@@ -544,10 +537,7 @@ public void testInsertRowThrottling() {
             1000000L * (parameterProvider.getInsertThrottleThresholdInPercentage() - 1) / 100);
 
     CompletableFuture<Void> future =
-        CompletableFuture.runAsync(
-            () -> {
-              channel.throttleInsertIfNeeded(mockedRunTime);
-            });
+        CompletableFuture.runAsync(() -> channel.throttleInsertIfNeeded(mockedRunTime));
 
     try {
       future.get(1L, TimeUnit.SECONDS);
@@ -560,10 +550,10 @@ public void testInsertRowThrottling() {
 
   @Test
   public void testFlush() throws Exception {
-    SnowflakeStreamingIngestClientInternal client =
-        Mockito.spy(new SnowflakeStreamingIngestClientInternal("client"));
-    SnowflakeStreamingIngestChannelInternal channel =
-        new SnowflakeStreamingIngestChannelInternal(
+    SnowflakeStreamingIngestClientInternal<?> client =
+        Mockito.spy(new SnowflakeStreamingIngestClientInternal<>("client"));
+    SnowflakeStreamingIngestChannelInternal<?> channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel",
             "db",
             "schema",
@@ -574,8 +564,7 @@ public void testFlush() throws Exception {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
     ChannelsStatusResponse response = new ChannelsStatusResponse();
     response.setStatusCode(0L);
     response.setMessage("Success");
@@ -596,10 +585,10 @@ public void testFlush() throws Exception {
 
   @Test
   public void testClose() throws Exception {
-    SnowflakeStreamingIngestClientInternal client =
-        Mockito.spy(new SnowflakeStreamingIngestClientInternal("client"));
+    SnowflakeStreamingIngestClientInternal<?> client =
+        Mockito.spy(new SnowflakeStreamingIngestClientInternal<>("client"));
     SnowflakeStreamingIngestChannel channel =
-        new SnowflakeStreamingIngestChannelInternal(
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel",
             "db",
             "schema",
@@ -610,8 +599,7 @@ public void testClose() throws Exception {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
     ChannelsStatusResponse response = new ChannelsStatusResponse();
     response.setStatusCode(0L);
     response.setMessage("Success");
@@ -630,10 +618,10 @@ public void testClose() throws Exception {
   @Test
   public void testGetLatestCommittedOffsetToken() {
     String offsetToken = "10";
-    SnowflakeStreamingIngestClientInternal client =
-        Mockito.spy(new SnowflakeStreamingIngestClientInternal("client"));
+    SnowflakeStreamingIngestClientInternal<?> client =
+        Mockito.spy(new SnowflakeStreamingIngestClientInternal<>("client"));
     SnowflakeStreamingIngestChannel channel =
-        new SnowflakeStreamingIngestChannelInternal(
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel",
             "db",
             "schema",
@@ -644,8 +632,7 @@ public void testGetLatestCommittedOffsetToken() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
 
     ChannelsStatusResponse response = new ChannelsStatusResponse();
     response.setStatusCode(0L);
@@ -654,7 +641,7 @@ public void testGetLatestCommittedOffsetToken() {
     // Test success case
     ChannelsStatusResponse.ChannelStatusResponseDTO channelStatus =
         new ChannelsStatusResponse.ChannelStatusResponseDTO();
-    channelStatus.setStatusCode((long) RESPONSE_SUCCESS);
+    channelStatus.setStatusCode(RESPONSE_SUCCESS);
     channelStatus.setPersistedOffsetToken(offsetToken);
     response.setChannels(Collections.singletonList(channelStatus));
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index 345e968aa..becbde047 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -50,13 +50,14 @@
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Pair;
 import net.snowflake.ingest.utils.ParameterProvider;
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.SnowflakeURL;
 import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.memory.RootAllocator;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -66,10 +67,10 @@
 public class SnowflakeStreamingIngestClientTest {
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
-  SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel1;
-  SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel2;
-  SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel3;
-  SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel4;
+  SnowflakeStreamingIngestChannelInternal<StubChunkData> channel1;
+  SnowflakeStreamingIngestChannelInternal<StubChunkData> channel2;
+  SnowflakeStreamingIngestChannelInternal<StubChunkData> channel3;
+  SnowflakeStreamingIngestChannelInternal<StubChunkData> channel4;
 
   @Before
   public void setup() {
@@ -88,7 +89,8 @@ public void setup() {
             "key",
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            Constants.BdecVersion.ONE,
+            new RootAllocator());
     channel2 =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel2",
@@ -102,7 +104,8 @@ public void setup() {
             "key",
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            Constants.BdecVersion.ONE,
+            new RootAllocator());
     channel3 =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel3",
@@ -116,7 +119,8 @@ public void setup() {
             "key",
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            Constants.BdecVersion.ONE,
+            new RootAllocator());
     channel4 =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel4",
@@ -130,7 +134,8 @@ public void setup() {
             "key",
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            Constants.BdecVersion.ONE,
+            new RootAllocator());
   }
 
   @Test
@@ -148,7 +153,10 @@ public void testConstructorParameters() throws Exception {
 
     SnowflakeStreamingIngestClientInternal<?> client =
         (SnowflakeStreamingIngestClientInternal<?>)
-            SnowflakeStreamingIngestClientFactory.builder("client").setProperties(prop).build();
+            SnowflakeStreamingIngestClientFactory.builder("client")
+                .setProperties(prop)
+                .setParameterOverrides(parameterMap)
+                .build();
 
     Assert.assertEquals("client", client.getName());
     Assert.assertEquals(123, client.getParameterProvider().getBufferFlushIntervalInMs());
@@ -190,8 +198,7 @@ public void testClientFactoryMissingName() throws Exception {
     prop.put(ROLE, "role");
 
     try {
-      SnowflakeStreamingIngestClient client =
-          SnowflakeStreamingIngestClientFactory.builder("client").setProperties(prop).build();
+      SnowflakeStreamingIngestClientFactory.builder("client").setProperties(prop).build();
       Assert.fail("Client creation should fail.");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.MISSING_CONFIG.getMessageCode(), e.getVendorCode());
@@ -206,8 +213,7 @@ public void testClientFactoryMissingURL() throws Exception {
     prop.put(ROLE, "role");
 
     try {
-      SnowflakeStreamingIngestClient client =
-          SnowflakeStreamingIngestClientFactory.builder("client").setProperties(prop).build();
+      SnowflakeStreamingIngestClientFactory.builder("client").setProperties(prop).build();
       Assert.fail("Client creation should fail.");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.MISSING_CONFIG.getMessageCode(), e.getVendorCode());
@@ -223,8 +229,7 @@ public void testClientFactoryInvalidURL() throws Exception {
     prop.put(ROLE, "role");
 
     try {
-      SnowflakeStreamingIngestClient client =
-          SnowflakeStreamingIngestClientFactory.builder("client").setProperties(prop).build();
+      SnowflakeStreamingIngestClientFactory.builder("client").setProperties(prop).build();
       Assert.fail("Client creation should fail.");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.INVALID_URL.getMessageCode(), e.getVendorCode());
@@ -240,8 +245,7 @@ public void testClientFactoryInvalidPrivateKey() throws Exception {
     prop.put(ROLE, "role");
 
     try {
-      SnowflakeStreamingIngestClient client =
-          SnowflakeStreamingIngestClientFactory.builder("client").setProperties(prop).build();
+      SnowflakeStreamingIngestClientFactory.builder("client").setProperties(prop).build();
       Assert.fail("Client creation should fail.");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.INVALID_PRIVATE_KEY.getMessageCode(), e.getVendorCode());
@@ -304,7 +308,7 @@ private String generateAESKey(PrivateKey key, char[] passwd)
   public void testGetChannelsStatusWithRequest() throws Exception {
     ChannelsStatusResponse.ChannelStatusResponseDTO channelStatus =
         new ChannelsStatusResponse.ChannelStatusResponseDTO();
-    channelStatus.setStatusCode((long) RESPONSE_SUCCESS);
+    channelStatus.setStatusCode(RESPONSE_SUCCESS);
     ChannelsStatusResponse response = new ChannelsStatusResponse();
     response.setStatusCode(0L);
     response.setMessage("honk");
@@ -347,7 +351,8 @@ public void testGetChannelsStatusWithRequest() throws Exception {
             "key",
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            Constants.BdecVersion.ONE,
+            new RootAllocator());
 
     ChannelsStatusRequest.ChannelStatusRequestDTO dto =
         new ChannelsStatusRequest.ChannelStatusRequestDTO(channel);
@@ -405,7 +410,8 @@ public void testGetChannelsStatusWithRequestError() throws Exception {
             "key",
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            Constants.BdecVersion.ONE,
+            new RootAllocator());
 
     try {
       client.getChannelsStatus(Collections.singletonList(channel));
@@ -443,7 +449,8 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
             "key",
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            Constants.BdecVersion.ONE,
+            new RootAllocator());
 
     ChannelMetadata channelMetadata =
         ChannelMetadata.builder()
@@ -480,8 +487,7 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
             payload, REGISTER_BLOB_ENDPOINT, "register blob");
 
     Assert.assertEquals(
-        String.format("%s%s", urlStr, REGISTER_BLOB_ENDPOINT),
-        request.getRequestLine().getUri().toString());
+        String.format("%s%s", urlStr, REGISTER_BLOB_ENDPOINT), request.getRequestLine().getUri());
     Assert.assertNotNull(request.getFirstHeader(HttpHeaders.USER_AGENT));
     Assert.assertNotNull(request.getFirstHeader(HttpHeaders.AUTHORIZATION));
     Assert.assertEquals("POST", request.getMethod());
@@ -588,7 +594,7 @@ private Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> getRetryBlobMetadata(
     badChunkRegisterStatus.setTableName(chunkMetadata1.getTableName());
     badChunkRegisterStatus.setChannelsStatus(channelRegisterStatuses);
     badChunks.add(badChunkRegisterStatus);
-    return new Pair(blobs, badChunks);
+    return new Pair<>(blobs, badChunks);
   }
 
   @Test
@@ -618,7 +624,7 @@ public void testGetRetryBlobs() throws Exception {
     Assert.assertEquals(
         Sets.newHashSet("channel1", "channel2"),
         result.get(0).getChunks().get(0).getChannels().stream()
-            .map(c -> c.getChannelName())
+            .map(ChannelMetadata::getChannelName)
             .collect(Collectors.toSet()));
     Assert.assertEquals(ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT, result.get(0).getVersion());
   }
@@ -650,8 +656,7 @@ public void testRegisterBlobErrorResponse() throws Exception {
 
     try {
       List<BlobMetadata> blobs =
-          Collections.singletonList(
-              new BlobMetadata("path", "md5", new ArrayList<ChunkMetadata>()));
+          Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
       client.registerBlobs(blobs);
       Assert.fail("Register blob should fail on 404 error");
     } catch (SFException e) {
@@ -698,8 +703,7 @@ public void testRegisterBlobSnowflakeInternalErrorResponse() throws Exception {
 
     try {
       List<BlobMetadata> blobs =
-          Collections.singletonList(
-              new BlobMetadata("path", "md5", new ArrayList<ChunkMetadata>()));
+          Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
       client.registerBlobs(blobs);
       Assert.fail("Register blob should fail on SF internal error");
     } catch (SFException e) {
@@ -754,7 +758,7 @@ public void testRegisterBlobSuccessResponse() throws Exception {
             null);
 
     List<BlobMetadata> blobs =
-        Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<ChunkMetadata>()));
+        Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
     client.registerBlobs(blobs);
   }
 
@@ -794,7 +798,7 @@ public void testRegisterBlobsRetries() throws Exception {
 
     List<BlobRegisterStatus> blobRegisterStatuses = new ArrayList<>();
     BlobRegisterStatus blobRegisterStatus1 = new BlobRegisterStatus();
-    blobRegisterStatus1.setChunksStatus(badChunks.stream().collect(Collectors.toList()));
+    blobRegisterStatus1.setChunksStatus(new ArrayList<>(badChunks));
     blobRegisterStatuses.add(blobRegisterStatus1);
     BlobRegisterStatus blobRegisterStatus2 = new BlobRegisterStatus();
     blobRegisterStatus2.setChunksStatus(Collections.singletonList(goodChunkRegisterStatus));
@@ -827,7 +831,7 @@ public void testRegisterBlobsRetries() throws Exception {
     RequestBuilder requestBuilder =
         Mockito.spy(
             new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair()));
-    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
+    SnowflakeStreamingIngestClientInternal<StubChunkData> client =
         new SnowflakeStreamingIngestClientInternal<>(
             "client",
             new SnowflakeURL("snowflake.dev.local:8082"),
@@ -910,7 +914,7 @@ public void testRegisterBlobsRetriesSucceeds() throws Exception {
 
     List<BlobRegisterStatus> blobRegisterStatuses = new ArrayList<>();
     BlobRegisterStatus blobRegisterStatus1 = new BlobRegisterStatus();
-    blobRegisterStatus1.setChunksStatus(badChunks.stream().collect(Collectors.toList()));
+    blobRegisterStatus1.setChunksStatus(new ArrayList<>(badChunks));
     blobRegisterStatuses.add(blobRegisterStatus1);
     BlobRegisterStatus blobRegisterStatus2 = new BlobRegisterStatus();
     blobRegisterStatus2.setChunksStatus(Collections.singletonList(goodChunkRegisterStatus));
@@ -942,7 +946,7 @@ public void testRegisterBlobsRetriesSucceeds() throws Exception {
     RequestBuilder requestBuilder =
         Mockito.spy(
             new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair()));
-    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
+    SnowflakeStreamingIngestClientInternal<StubChunkData> client =
         new SnowflakeStreamingIngestClientInternal<>(
             "client",
             new SnowflakeURL("snowflake.dev.local:8082"),
@@ -1016,8 +1020,8 @@ public void testRegisterBlobResponseWithInvalidChannel() throws Exception {
 
     RequestBuilder requestBuilder =
         new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair());
-    SnowflakeStreamingIngestClientInternal client =
-        new SnowflakeStreamingIngestClientInternal(
+    SnowflakeStreamingIngestClientInternal<StubChunkData> client =
+        new SnowflakeStreamingIngestClientInternal<>(
             "client",
             new SnowflakeURL("snowflake.dev.local:8082"),
             null,
@@ -1026,8 +1030,8 @@ public void testRegisterBlobResponseWithInvalidChannel() throws Exception {
             requestBuilder,
             null);
 
-    SnowflakeStreamingIngestChannelInternal channel1 =
-        new SnowflakeStreamingIngestChannelInternal(
+    SnowflakeStreamingIngestChannelInternal<StubChunkData> channel1 =
+        new SnowflakeStreamingIngestChannelInternal<>(
             channel1Name,
             dbName,
             schemaName,
@@ -1038,10 +1042,9 @@ public void testRegisterBlobResponseWithInvalidChannel() throws Exception {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
-    SnowflakeStreamingIngestChannelInternal channel2 =
-        new SnowflakeStreamingIngestChannelInternal(
+            OpenChannelRequest.OnErrorOption.CONTINUE);
+    SnowflakeStreamingIngestChannelInternal<StubChunkData> channel2 =
+        new SnowflakeStreamingIngestChannelInternal<>(
             channel2Name,
             dbName,
             schemaName,
@@ -1052,8 +1055,7 @@ public void testRegisterBlobResponseWithInvalidChannel() throws Exception {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
     client.getChannelCache().addChannel(channel1);
     client.getChannelCache().addChannel(channel2);
 
@@ -1061,7 +1063,7 @@ public void testRegisterBlobResponseWithInvalidChannel() throws Exception {
     Assert.assertTrue(channel2.isValid());
 
     List<BlobMetadata> blobs =
-        Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<ChunkMetadata>()));
+        Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
     client.registerBlobs(blobs);
 
     // Channel2 should be invalidated now
@@ -1071,8 +1073,8 @@ public void testRegisterBlobResponseWithInvalidChannel() throws Exception {
 
   @Test
   public void testFlush() throws Exception {
-    SnowflakeStreamingIngestClientInternal client =
-        Mockito.spy(new SnowflakeStreamingIngestClientInternal("client"));
+    SnowflakeStreamingIngestClientInternal<?> client =
+        Mockito.spy(new SnowflakeStreamingIngestClientInternal<>("client"));
     ChannelsStatusResponse response = new ChannelsStatusResponse();
     response.setStatusCode(0L);
     response.setMessage("Success");
@@ -1093,8 +1095,8 @@ public void testFlush() throws Exception {
 
   @Test
   public void testClose() throws Exception {
-    SnowflakeStreamingIngestClientInternal client =
-        Mockito.spy(new SnowflakeStreamingIngestClientInternal("client"));
+    SnowflakeStreamingIngestClientInternal<?> client =
+        Mockito.spy(new SnowflakeStreamingIngestClientInternal<>("client"));
     ChannelsStatusResponse response = new ChannelsStatusResponse();
     response.setStatusCode(0L);
     response.setMessage("Success");
@@ -1127,12 +1129,8 @@ public void testClose() throws Exception {
 
   @Test
   public void testCloseWithError() throws Exception {
-    SnowflakeStreamingIngestClientInternal client =
-        Mockito.spy(new SnowflakeStreamingIngestClientInternal("client"));
-    ChannelsStatusResponse response = new ChannelsStatusResponse();
-    response.setStatusCode(20L);
-    response.setMessage("Failure");
-    response.setChannels(new ArrayList<>());
+    SnowflakeStreamingIngestClientInternal<?> client =
+        Mockito.spy(new SnowflakeStreamingIngestClientInternal<>("client"));
 
     CompletableFuture<Void> future = new CompletableFuture<>();
     future.completeExceptionally(new Exception("Simulating Error"));
@@ -1169,10 +1167,10 @@ public void testCloseWithError() throws Exception {
 
   @Test
   public void testVerifyChannelsAreFullyCommittedSuccess() throws Exception {
-    SnowflakeStreamingIngestClientInternal client =
-        Mockito.spy(new SnowflakeStreamingIngestClientInternal("client"));
-    SnowflakeStreamingIngestChannelInternal channel =
-        new SnowflakeStreamingIngestChannelInternal(
+    SnowflakeStreamingIngestClientInternal<StubChunkData> client =
+        Mockito.spy(new SnowflakeStreamingIngestClientInternal<>("client"));
+    SnowflakeStreamingIngestChannelInternal<StubChunkData> channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
             "channel1",
             "db",
             "schema",
@@ -1183,8 +1181,7 @@ public void testVerifyChannelsAreFullyCommittedSuccess() throws Exception {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE,
-            true);
+            OpenChannelRequest.OnErrorOption.CONTINUE);
     client.getChannelCache().addChannel(channel);
 
     ChannelsStatusResponse response = new ChannelsStatusResponse();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StubChunkData.java b/src/test/java/net/snowflake/ingest/streaming/internal/StubChunkData.java
new file mode 100644
index 000000000..21edab33a
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StubChunkData.java
@@ -0,0 +1,4 @@
+package net.snowflake.ingest.streaming.internal;
+
+/** Stub class for class parameters where it does not matter. */
+public final class StubChunkData {}

From d61f95778f6f2140f963a4f13f14f22547fd94d0 Mon Sep 17 00:00:00 2001
From: sfc-gh-snyk-sca-sa
 <95290361+sfc-gh-snyk-sca-sa@users.noreply.github.com>
Date: Fri, 16 Sep 2022 22:38:02 +0530
Subject: [PATCH 061/356] SNOW-470176: [Snyk] Fix for 2 vulnerabilities (#225)

fix: pom.xml to reduce vulnerabilities

The following vulnerabilities are fixed with an upgrade:
- https://snyk.io/vuln/SNYK-JAVA-COMMONSCODEC-561518
- https://snyk.io/vuln/SNYK-JAVA-IONETTY-2812456

Co-authored-by: snyk-bot <snyk-bot@snyk.io>
---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 75969b9bf..9f377c7ca 100644
--- a/pom.xml
+++ b/pom.xml
@@ -42,7 +42,7 @@
         <shadeBase>net.snowflake.ingest.internal</shadeBase>
         <jacoco.skip.instrument>true</jacoco.skip.instrument>
         <jacoco.version>0.8.5</jacoco.version>
-        <arrow.version>8.0.0</arrow.version>
+        <arrow.version>9.0.0</arrow.version>
     </properties>
 
     <build>
@@ -397,7 +397,7 @@
                                 </filter>
                             </filters>
                             <transformers>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                 </transformer>
                             </transformers>

From a0ac0449e814186ecf5a55141162f50c410c908f Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Mon, 19 Sep 2022 21:59:21 +0200
Subject: [PATCH 062/356] SNOW-655614 Data type validation: Boolean (#228)

SNOW-655614 Data type validation: Boolean

This is the first commit in a series of commits focusing on
comprehensive data type validation. It establishes testing
infrastructure and adds tests for BOOLEAN data type.

It also makes sure that negative integers are represented
as TRUE.
---
 .pre-commit-config.yaml                       |   2 +-
 .../internal/DataValidationUtil.java          |  42 ++-
 .../java/net/snowflake/ingest/TestUtils.java  |  33 +-
 .../internal/DataValidationUtilTest.java      |  64 ++--
 .../datatypes/AbstractDataTypeTest.java       | 325 ++++++++++++++++++
 .../internal/datatypes/LogicalTypesIT.java    |  73 ++++
 .../internal/datatypes/Provider.java          |  89 +++++
 7 files changed, 576 insertions(+), 52 deletions(-)
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/datatypes/Provider.java

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0809b2dac..6d3ea2497 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -2,4 +2,4 @@ repos:
 - hooks:
   - id: secret-scanner
   repo: git@github.com:snowflakedb/casec_precommit.git
-  rev: v1.5
+  rev: v1.8
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index f82f9862f..ba01e5681 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -26,6 +26,7 @@
 class DataValidationUtil {
   static final int MAX_STRING_LENGTH = 16 * 1024 * 1024;
   static final int MAX_BINARY_LENGTH = 8 * 1024 * 1024;
+
   static final BigInteger MAX_BIGINTEGER = BigInteger.valueOf(10).pow(38);
   static final BigInteger MIN_BIGINTEGER =
       BigInteger.valueOf(-1).multiply(BigInteger.valueOf(10).pow(38));
@@ -665,24 +666,24 @@ static double validateAndParseReal(Object input) {
    * @return 1 or 0 where 1=true, 0=false
    */
   static int validateAndParseBoolean(Object input) {
-    int output = 0;
     if (input instanceof Boolean) {
-      output = (boolean) input ? 1 : 0;
+      return (boolean) input ? 1 : 0;
     } else if (input instanceof Number) {
-      output = ((Number) input).doubleValue() > 0 ? 1 : 0;
-    } else {
-      output = convertStringToBoolean((String) input) ? 1 : 0;
+      return new BigDecimal(input.toString()).compareTo(BigDecimal.ZERO) == 0 ? 0 : 1;
+    } else if (input instanceof String) {
+      return convertStringToBoolean((String) input) ? 1 : 0;
     }
-    return output;
+
+    throw cannotConvertException(input, "boolean");
   }
 
   static Set<String> allowedBooleanStringsLowerCased =
       Sets.newHashSet("1", "0", "yes", "no", "y", "n", "t", "f", "true", "false", "on", "off");
 
-  static boolean convertStringToBoolean(String value) {
+  private static boolean convertStringToBoolean(String value) {
     String lowerCasedValue = value.toLowerCase();
     if (!allowedBooleanStringsLowerCased.contains(lowerCasedValue)) {
-      throw new SFException(ErrorCode.INVALID_ROW, value);
+      throw cannotConvertException(value, "boolean");
     }
     return "1".equals(lowerCasedValue)
         || "yes".equals(lowerCasedValue)
@@ -691,4 +692,29 @@ static boolean convertStringToBoolean(String value) {
         || "true".equals(lowerCasedValue)
         || "on".equals(lowerCasedValue);
   }
+
+  /**
+   * Create exception when value cannot be ingested into column of a specific type.
+   *
+   * @param value Invalid value causing the exception
+   * @param type Snowflake column type
+   */
+  private static SFException cannotConvertException(Object value, String type) {
+    return new SFException(
+        ErrorCode.INVALID_ROW,
+        sanitizeValueForExceptionMessage(value),
+        String.format("Value cannot be converted to %s", type));
+  }
+
+  /**
+   * Before passing a value to an exception string, we should limit how many characters are
+   * displayed. Important especially for "value exceeds max column size" exceptions.
+   *
+   * @param value Value to adapt for exception message
+   */
+  private static String sanitizeValueForExceptionMessage(Object value) {
+    int maxSize = 20;
+    String valueString = value.toString();
+    return valueString.length() <= maxSize ? valueString : valueString.substring(0, 20) + "...";
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 95827d5df..0e2d3f145 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -34,8 +34,10 @@
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.utils.Utils;
 import org.apache.commons.codec.binary.Base64;
+import org.junit.Assert;
 
 public class TestUtils {
   // profile path, follow readme for the format
@@ -222,8 +224,12 @@ public static Connection getConnection() throws Exception {
    * @throws Exception
    */
   public static Connection getConnection(boolean isStreamingConnection) throws Exception {
-    if (!isStreamingConnection && snowpipeConn != null) return snowpipeConn;
-    if (isStreamingConnection && streamingConn != null) return streamingConn;
+    if (!isStreamingConnection && snowpipeConn != null && !streamingConn.isClosed()) {
+      return snowpipeConn;
+    }
+    if (isStreamingConnection && streamingConn != null && !streamingConn.isClosed()) {
+      return streamingConn;
+    }
 
     if (profile == null) init();
     // check first to see if we have the Snowflake JDBC
@@ -317,4 +323,27 @@ public static SimpleIngestManager getManagerUsingBuilderPattern(
         .setUserAgentSuffix(userAgentSuffix)
         .build();
   }
+
+  /**
+   * Given a channel and expected offset, this method waits up to 60 seconds until the last
+   * committed offset is equal to the passed offset
+   */
+  public static void waitForOffset(SnowflakeStreamingIngestChannel channel, String expectedOffset)
+      throws InterruptedException {
+    int counter = 0;
+    String lastCommittedOffset = null;
+    while (counter < 600) {
+      String currentOffset = channel.getLatestCommittedOffsetToken();
+      if (expectedOffset.equals(currentOffset)) {
+        return;
+      }
+      lastCommittedOffset = currentOffset;
+      counter++;
+      Thread.sleep(100);
+    }
+    Assert.fail(
+        String.format(
+            "Timeout exceeded while waiting for offset %s. Last committed offset: %s",
+            expectedOffset, lastCommittedOffset));
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 3e082127f..99e0aae3e 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -1,6 +1,8 @@
 package net.snowflake.ingest.streaming.internal;
 
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.MAX_BIGINTEGER;
+import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseBoolean;
+import static org.junit.Assert.assertEquals;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -25,17 +27,16 @@ public class DataValidationUtilTest {
   private static final Object[] goodIntegersValue10 =
       new Object[] {10D, 10F, 10L, new BigInteger("10"), 10, "10", "1e1", "1.0e1"};
 
-  private static final Object[] trueBooleanInput =
-      new Object[] {true, "true", "True", "TruE", "t", "yes", "YeS", "y", "on", "1", 1.1};
-  private static final Object[] falseBooleanInput =
-      new Object[] {false, "false", "False", "FalsE", "f", "no", "NO", "n", "off", "0", 0};
+  private void expectError(ErrorCode expectedErrorCode, Function func, Object args) {
+    expectError(expectedErrorCode, () -> func.apply(args));
+  }
 
-  private void expectError(ErrorCode expectedError, Function func, Object args) {
+  private void expectError(ErrorCode expectedErrorCode, Runnable action) {
     try {
-      func.apply(args);
+      action.run();
       Assert.fail("Expected Exception");
     } catch (SFException e) {
-      Assert.assertEquals(expectedError.getMessageCode(), e.getVendorCode());
+      assertEquals(expectedErrorCode.getMessageCode(), e.getVendorCode());
     } catch (Exception e) {
       Assert.fail("Invalid error through");
     }
@@ -529,44 +530,25 @@ public void testValidateAndParseReal() throws Exception {
   }
 
   @Test
-  public void testValidateAndParseBoolean() throws Exception {
-    for (Object input : trueBooleanInput) {
-      Assert.assertEquals(1, DataValidationUtil.validateAndParseBoolean(input));
-    }
-    for (Object input : falseBooleanInput) {
-      Assert.assertEquals(0, DataValidationUtil.validateAndParseBoolean(input));
-    }
+  public void testValidateAndParseBoolean() {
 
-    // Error states
-    try {
-      DataValidationUtil.validateAndParseBoolean("honk");
-      Assert.fail("Expected invalid row error");
-    } catch (SFException err) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), err.getVendorCode());
+    for (Object input :
+        Arrays.asList(
+            true, "true", "True", "TruE", "t", "yes", "YeS", "y", "on", "1", 1.1, -1.1, -10, 10)) {
+      assertEquals(1, validateAndParseBoolean(input));
     }
-  }
 
-  @Test
-  public void testConvertStringToBoolean() throws Exception {
-    for (Object input : trueBooleanInput) {
-      if (input instanceof String) {
-        Assert.assertEquals(true, DataValidationUtil.convertStringToBoolean((String) input));
-      }
-    }
-
-    for (Object input : falseBooleanInput) {
-      if (input instanceof String) {
-        Assert.assertEquals(false, DataValidationUtil.convertStringToBoolean((String) input));
-      }
+    for (Object input :
+        Arrays.asList(false, "false", "False", "FalsE", "f", "no", "NO", "n", "off", "0", 0)) {
+      assertEquals(0, validateAndParseBoolean(input));
     }
 
-    try {
-      DataValidationUtil.convertStringToBoolean("honk");
-      Assert.fail("Expected error");
-    } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
-    } catch (Exception e) {
-      Assert.fail();
-    }
+    // Test forbidden values
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBoolean, new Object());
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBoolean, 't');
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBoolean, 'f');
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBoolean, new int[] {});
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBoolean, "foobar");
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBoolean, "");
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
new file mode 100644
index 000000000..a5e59d68c
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -0,0 +1,325 @@
+package net.snowflake.ingest.streaming.internal.datatypes;
+
+import static net.snowflake.ingest.utils.Constants.ROLE;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.function.Predicate;
+import net.snowflake.ingest.TestUtils;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import net.snowflake.ingest.utils.SFException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+
+public abstract class AbstractDataTypeTest {
+  private static final String SOURCE_COLUMN_NAME = "source";
+  private static final String VALUE_COLUMN_NAME = "value";
+
+  private static final String SOURCE_STREAMING_INGEST = "STREAMING_INGEST";
+  private static final String SOURCE_JDBC = "JDBC";
+  private static final String SCHEMA_NAME = "PUBLIC";
+
+  protected static BigInteger MAX_ALLOWED_BIG_INTEGER =
+      new BigInteger("99999999999999999999999999999999999999");
+  protected static BigInteger MIN_ALLOWED_BIG_INTEGER =
+      new BigInteger("-99999999999999999999999999999999999999");
+
+  protected static BigDecimal MAX_ALLOWED_BIG_DECIMAL = new BigDecimal(MAX_ALLOWED_BIG_INTEGER);
+  protected static BigDecimal MIN_ALLOWED_BIG_DECIMAL = new BigDecimal(MIN_ALLOWED_BIG_INTEGER);
+
+  protected Connection conn;
+  private String databaseName;
+
+  private String schemaName = "PUBLIC";
+  private SnowflakeStreamingIngestClient client;
+
+  protected String randomString() {
+    return UUID.randomUUID().toString().replace("-", "_");
+  }
+
+  private static final ObjectMapper objectMapper = new ObjectMapper();
+
+  @Before
+  public void before() throws Exception {
+    databaseName = String.format("SDK_DATATYPE_COMPATIBILITY_IT_%s", randomString());
+    conn = TestUtils.getConnection(true);
+    conn.createStatement().execute(String.format("create or replace database %s;", databaseName));
+    conn.createStatement().execute(String.format("use database %s;", databaseName));
+    conn.createStatement().execute(String.format("use schema %s;", schemaName));
+
+    // Set to a random time zone not to interfere with any of the tests
+    conn.createStatement().execute("alter session set timezone = 'America/New_York';");
+
+    conn.createStatement().execute(String.format("use warehouse %s;", TestUtils.getWarehouse()));
+
+    Properties props = TestUtils.getProperties();
+    if (props.getProperty(ROLE).equals("DEFAULT_ROLE")) {
+      props.setProperty(ROLE, "ACCOUNTADMIN");
+    }
+    client = SnowflakeStreamingIngestClientFactory.builder("client1").setProperties(props).build();
+  }
+
+  @After
+  public void after() throws Exception {
+    conn.createStatement().executeQuery(String.format("drop database %s", databaseName));
+    if (client != null) {
+      client.close();
+    }
+    if (conn != null) {
+      conn.close();
+    }
+  }
+
+  protected String createTable(String dataType) throws SQLException {
+    String tableName =
+        String.format("test_%s_%s", dataType, UUID.randomUUID())
+            .replace('-', '_')
+            .replace(' ', '_')
+            .replace('(', '_')
+            .replace(')', '_')
+            .replace(',', '_');
+
+    //    System.out.printf("Creating table %s.%s.%s%n", databaseName, schemaName, tableName);
+    conn.createStatement()
+        .execute(
+            String.format(
+                "create or replace table %s (%s string, %s %s)",
+                tableName, SOURCE_COLUMN_NAME, VALUE_COLUMN_NAME, dataType));
+    return tableName;
+  }
+
+  protected SnowflakeStreamingIngestChannel openChannel(String tableName) {
+    OpenChannelRequest openChannelRequest =
+        OpenChannelRequest.builder("CHANNEL")
+            .setDBName(databaseName)
+            .setSchemaName(SCHEMA_NAME)
+            .setTableName(tableName)
+            .setOnErrorOption(OpenChannelRequest.OnErrorOption.ABORT)
+            .build();
+    return client.openChannel(openChannelRequest);
+  }
+
+  private Map<String, Object> createStreamingIngestRow(Object value) {
+    Map<String, Object> row = new HashMap<>();
+    row.put(SOURCE_COLUMN_NAME, SOURCE_STREAMING_INGEST);
+    row.put(VALUE_COLUMN_NAME, value);
+    return row;
+  }
+
+  private <T> void expectError(String dataType, T value, Predicate<Exception> errorMatcher)
+      throws Exception {
+    String tableName = createTable(dataType);
+    SnowflakeStreamingIngestChannel channel = null;
+    try {
+      channel = openChannel(tableName);
+      channel.insertRow(createStreamingIngestRow(value), "0");
+      Assert.fail(
+          String.format("Inserting value %s for data type %s should fail", value, dataType));
+    } catch (Exception e) {
+      if (errorMatcher.test(e)) {
+        // all good, expected exception has been thrown
+      } else {
+        e.printStackTrace();
+        Assert.fail(String.format("Unexpected exception thrown: %s", e.getMessage()));
+      }
+    } finally {
+      if (channel != null) {
+        channel.close().get();
+      }
+    }
+  }
+
+  protected <T> void expectNumberOutOfRangeError(
+      String dataType, T value, int maxPowerOf10Exclusive) throws Exception {
+    expectError(
+        dataType,
+        value,
+        x ->
+            (x instanceof SFException
+                && x.getMessage()
+                    .contains(
+                        String.format(
+                            "Number out of representable exclusive range of (-1e%d..1e%d)",
+                            maxPowerOf10Exclusive, maxPowerOf10Exclusive))));
+  }
+
+  protected <T> void expectArrowNotSupported(String dataType, T value) throws Exception {
+    expectError(
+        dataType,
+        value,
+        x ->
+            (x instanceof SFException
+                && x.getMessage().contains("The given row cannot be converted to Arrow format")));
+  }
+
+  /**
+   * Simplified version, which does not insert using JDBC. Useful for testing non-JDBC types like
+   * BigInteger, java.time.* types, etc.
+   */
+  <STREAMING_INGEST_WRITE, JDBC_READ> void testIngestion(
+      String dataType,
+      STREAMING_INGEST_WRITE streamingIngestWriteValue,
+      JDBC_READ expectedValue,
+      Provider<JDBC_READ> selectProvider)
+      throws Exception {
+    ingestAndAssert(dataType, streamingIngestWriteValue, null, expectedValue, null, selectProvider);
+  }
+
+  /**
+   * Simplified version where streaming ingest write type, JDBC write type and JDBC read type are
+   * the same type
+   */
+  <T> void testJdbcTypeCompatibility(String typeName, T value, Provider<T> provider)
+      throws Exception {
+    ingestAndAssert(typeName, value, value, value, provider, provider);
+  }
+
+  /** Simplified version where write value for streaming ingest and JDBC are the same */
+  <WRITE, READ> void testJdbcTypeCompatibility(
+      String typeName,
+      WRITE writeValue,
+      READ expectedValue,
+      Provider<WRITE> insertProvider,
+      Provider<READ> selectProvider)
+      throws Exception {
+    ingestAndAssert(
+        typeName, writeValue, writeValue, expectedValue, insertProvider, selectProvider);
+  }
+
+  /**
+   * Ingests values with streaming ingest and JDBC driver, SELECTs them back with WHERE condition
+   * and asserts they exist.
+   *
+   * @param dataType Snowflake data type
+   * @param streamingIngestWriteValue Value ingested by streaming ingest
+   * @param jdbcWriteValue Value written by JDBC driver
+   * @param expectedValue Expected value received from JDBC driver SELECT
+   * @param insertProvider JDBC parameter provider for INSERT
+   * @param selectProvider JDBC parameter provider for SELECT ... WHERE
+   * @param <STREAMING_INGEST_WRITE> Type ingested by streaming ingest
+   * @param <JDBC_WRITE> Type written by JDBC driver
+   * @param <JDBC_READ> Type read by JDBC driver
+   */
+  <STREAMING_INGEST_WRITE, JDBC_WRITE, JDBC_READ> void ingestAndAssert(
+      String dataType,
+      STREAMING_INGEST_WRITE streamingIngestWriteValue,
+      JDBC_WRITE jdbcWriteValue,
+      JDBC_READ expectedValue,
+      Provider<JDBC_WRITE> insertProvider,
+      Provider<JDBC_READ> selectProvider)
+      throws Exception {
+    if (jdbcWriteValue == null ^ insertProvider == null)
+      throw new IllegalArgumentException(
+          "jdbcWriteValue and provider must be both null or not null");
+    boolean insertAlsoWithJdbc = jdbcWriteValue != null;
+    String tableName = createTable(dataType);
+    String offsetToken = UUID.randomUUID().toString();
+
+    // Insert using JDBC
+    if (insertAlsoWithJdbc) {
+      String query = "insert into %s (select ?, ?)";
+      PreparedStatement insertStatement = conn.prepareStatement(String.format(query, tableName));
+      insertStatement.setString(1, SOURCE_JDBC);
+      insertProvider.provide(insertStatement, 2, jdbcWriteValue);
+      insertStatement.execute();
+    }
+
+    // Ingest using streaming ingest
+    SnowflakeStreamingIngestChannel channel = openChannel(tableName);
+    channel.insertRow(createStreamingIngestRow(streamingIngestWriteValue), offsetToken);
+    TestUtils.waitForOffset(channel, offsetToken);
+
+    // Select expected value and assert that expected number of results is returned (selecting with
+    // WHERE also tests pruning).
+    // The following is evaluated as true in Snowflake:
+    //     select '2020-01-01 14:00:00 +0000'::timestamp_tz = '2020-01-01 12:00:00
+    // -0200'::timestamp_tz;
+    // Therefore, to correctly test TIMESTAMP_TZ values, we need to compare them as strings.
+    // We can do that for all date/time types, not just TIMESTAMP_TZ.
+    String selectQuery;
+    if (expectedValue == null) {
+      selectQuery =
+          String.format("select count(*) from %s where %s is NULL", tableName, VALUE_COLUMN_NAME);
+    } else if (dataType.startsWith("TIMESTAMP_")) {
+      selectQuery =
+          String.format(
+              "select count(*) from %s where to_varchar(%s, 'YYYY-MM-DD HH24:MI:SS.FF TZHTZM') ="
+                  + " ?;",
+              tableName, VALUE_COLUMN_NAME);
+    } else if (dataType.startsWith("TIME")) {
+      selectQuery =
+          String.format(
+              "select count(*) from %s where to_varchar(%s, 'HH24:MI:SS.FF TZHTZM') = ?;",
+              tableName, VALUE_COLUMN_NAME);
+    } else {
+      selectQuery = "select count(*) from %s where %s = ?";
+    }
+    PreparedStatement selectStatement =
+        conn.prepareStatement(String.format(selectQuery, tableName, VALUE_COLUMN_NAME));
+    if (expectedValue != null) {
+      selectProvider.provide(selectStatement, 1, expectedValue);
+    }
+    ResultSet resultSet = selectStatement.executeQuery();
+    Assert.assertTrue(resultSet.next());
+    int count = resultSet.getInt(1);
+    Assert.assertEquals(insertAlsoWithJdbc ? 2 : 1, count);
+  }
+
+  <STREAMING_INGEST_WRITE> void assertVariant(
+      String dataType,
+      STREAMING_INGEST_WRITE streamingIngestWriteValue,
+      String expectedValue,
+      String expectedType)
+      throws Exception {
+    assertVariant(dataType, streamingIngestWriteValue, expectedValue, expectedType, true);
+  }
+
+  <STREAMING_INGEST_WRITE> void assertVariant(
+      String dataType,
+      STREAMING_INGEST_WRITE streamingIngestWriteValue,
+      String expectedValue,
+      String expectedType,
+      boolean compareAsJsons)
+      throws Exception {
+
+    String tableName = createTable(dataType);
+    String offsetToken = UUID.randomUUID().toString();
+
+    // Ingest using streaming ingest
+    SnowflakeStreamingIngestChannel channel = openChannel(tableName);
+    channel.insertRow(createStreamingIngestRow(streamingIngestWriteValue), offsetToken);
+    TestUtils.waitForOffset(channel, offsetToken);
+
+    String query =
+        String.format(
+            "select %s, typeof(%s) from %s", VALUE_COLUMN_NAME, VALUE_COLUMN_NAME, tableName);
+    ResultSet resultSet = conn.createStatement().executeQuery(query);
+    int counter = 0;
+    String value = null;
+    String typeof = null;
+    while (resultSet.next()) {
+      counter++;
+      value = resultSet.getString(1);
+      typeof = resultSet.getString(2);
+    }
+
+    Assert.assertEquals(1, counter);
+    if (compareAsJsons)
+      Assert.assertEquals(objectMapper.readTree(expectedValue), objectMapper.readTree(value));
+    else Assert.assertEquals(expectedValue, value);
+    Assert.assertEquals(expectedType, typeof);
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
new file mode 100644
index 000000000..1fdd2ad5e
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
@@ -0,0 +1,73 @@
+package net.snowflake.ingest.streaming.internal.datatypes;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import org.junit.Test;
+
+public class LogicalTypesIT extends AbstractDataTypeTest {
+  @Test
+  public void testLogicalTypes() throws Exception {
+    // Test booleans
+    testJdbcTypeCompatibility("BOOLEAN", true, new BooleanProvider());
+    testJdbcTypeCompatibility("BOOLEAN", false, new BooleanProvider());
+
+    // Test strings
+    testJdbcTypeCompatibility("BOOLEAN", "on", true, new StringProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility("BOOLEAN", "off", false, new StringProvider(), new BooleanProvider());
+
+    // Test integers
+    testJdbcTypeCompatibility(
+        "BOOLEAN", Integer.MIN_VALUE, true, new IntProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility(
+        "BOOLEAN", Integer.MAX_VALUE, true, new IntProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility("BOOLEAN", 0, false, new IntProvider(), new BooleanProvider());
+
+    // Test longs
+    testJdbcTypeCompatibility(
+        "BOOLEAN", Long.MIN_VALUE, true, new LongProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility(
+        "BOOLEAN", Long.MAX_VALUE, true, new LongProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility("BOOLEAN", 0, false, new IntProvider(), new BooleanProvider());
+
+    // Test bytes
+    testJdbcTypeCompatibility(
+        "BOOLEAN", Byte.MIN_VALUE, true, new ByteProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility(
+        "BOOLEAN", Byte.MAX_VALUE, true, new ByteProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility("BOOLEAN", 0, false, new IntProvider(), new BooleanProvider());
+
+    // Test shorts
+    testJdbcTypeCompatibility(
+        "BOOLEAN", Short.MIN_VALUE, true, new ShortProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility(
+        "BOOLEAN", Short.MAX_VALUE, true, new ShortProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility("BOOLEAN", 0, false, new IntProvider(), new BooleanProvider());
+
+    // Test BigIntegers
+    testIngestion("BOOLEAN", MAX_ALLOWED_BIG_INTEGER, true, new BooleanProvider());
+    testIngestion("BOOLEAN", MIN_ALLOWED_BIG_INTEGER, true, new BooleanProvider());
+    testIngestion("BOOLEAN", BigInteger.ZERO, false, new BooleanProvider());
+
+    // Test BigDecimals
+    testJdbcTypeCompatibility(
+        "BOOLEAN", MAX_ALLOWED_BIG_DECIMAL, true, new BigDecimalProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility(
+        "BOOLEAN", MIN_ALLOWED_BIG_DECIMAL, true, new BigDecimalProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility(
+        "BOOLEAN", new BigDecimal("0.5"), true, new BigDecimalProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility(
+        "BOOLEAN", BigDecimal.ZERO, false, new BigDecimalProvider(), new BooleanProvider());
+
+    // Test floats
+    testJdbcTypeCompatibility("BOOLEAN", 0.5f, true, new FloatProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility("BOOLEAN", -0.5f, true, new FloatProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility("BOOLEAN", 0f, false, new FloatProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility("BOOLEAN", 0.000f, false, new FloatProvider(), new BooleanProvider());
+
+    // Test double
+    testJdbcTypeCompatibility("BOOLEAN", 0.5, true, new DoubleProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility("BOOLEAN", -0.5, true, new DoubleProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility("BOOLEAN", 0d, false, new DoubleProvider(), new BooleanProvider());
+    testJdbcTypeCompatibility("BOOLEAN", 0.000, false, new DoubleProvider(), new BooleanProvider());
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/Provider.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/Provider.java
new file mode 100644
index 000000000..1ad9ae7c3
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/Provider.java
@@ -0,0 +1,89 @@
+package net.snowflake.ingest.streaming.internal.datatypes;
+
+import java.math.BigDecimal;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Implementations of this class are able to provide parameter of a certain type into JDBC
+ * statement. It is used to abstract over JDBC statements working with various types in tests.
+ */
+interface Provider<T> {
+  void provide(PreparedStatement stmt, int parameterIndex, T value) throws SQLException;
+}
+
+class LongProvider implements Provider<Long> {
+  @Override
+  public void provide(PreparedStatement stmt, int parameterIndex, Long value) throws SQLException {
+    stmt.setLong(parameterIndex, value);
+  }
+}
+
+class IntProvider implements Provider<Integer> {
+  @Override
+  public void provide(PreparedStatement stmt, int parameterIndex, Integer value)
+      throws SQLException {
+    stmt.setInt(parameterIndex, value);
+  }
+}
+
+class ByteProvider implements Provider<Byte> {
+  @Override
+  public void provide(PreparedStatement stmt, int parameterIndex, Byte value) throws SQLException {
+    stmt.setByte(parameterIndex, value);
+  }
+}
+
+class ShortProvider implements Provider<Short> {
+  @Override
+  public void provide(PreparedStatement stmt, int parameterIndex, Short value) throws SQLException {
+    stmt.setShort(parameterIndex, value);
+  }
+}
+
+class BigDecimalProvider implements Provider<BigDecimal> {
+  @Override
+  public void provide(PreparedStatement stmt, int parameterIndex, BigDecimal value)
+      throws SQLException {
+    stmt.setBigDecimal(parameterIndex, value);
+  }
+}
+
+class StringProvider implements Provider<java.lang.String> {
+  @Override
+  public void provide(PreparedStatement stmt, int parameterIndex, String value)
+      throws SQLException {
+    stmt.setString(parameterIndex, value);
+  }
+}
+
+class FloatProvider implements Provider<Float> {
+  @Override
+  public void provide(PreparedStatement stmt, int parameterIndex, Float value) throws SQLException {
+    stmt.setFloat(parameterIndex, value);
+  }
+}
+
+class DoubleProvider implements Provider<Double> {
+  @Override
+  public void provide(PreparedStatement stmt, int parameterIndex, Double value)
+      throws SQLException {
+    stmt.setDouble(parameterIndex, value);
+  }
+}
+
+class BooleanProvider implements Provider<Boolean> {
+  @Override
+  public void provide(PreparedStatement stmt, int parameterIndex, Boolean value)
+      throws SQLException {
+    stmt.setBoolean(parameterIndex, value);
+  }
+}
+
+class ByteArrayProvider implements Provider<byte[]> {
+  @Override
+  public void provide(PreparedStatement stmt, int parameterIndex, byte[] value)
+      throws SQLException {
+    stmt.setBytes(parameterIndex, value);
+  }
+}

From e5b3017586e575fdb235ceb35b466f51616b35a7 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Tue, 20 Sep 2022 19:16:36 +0200
Subject: [PATCH 063/356] SNOW-655614 Data type validation: Date and Time
 (#229)

* SNOW-655614 Data type validation: Date and Time

Type validation for Snowflake data types DATE, TIME, TIMESTAMP_*.
It tries to achieve consistency with the behavior of Snowflake
Worksheets.

* Only limited set of Java data types is accepted
    * No longer try to parse the result of Object#toString() call
* Never interpret any input in our code, always delegate the
  parsing logic to SnowflakeDateTimeFormat
* Numeric values are not allowed for any column (e.g. int 123)
* Decimal strings are not allowed (e.g. "1663580793.123")
* Numeric strings (e.g. "123") are still allowed because Snowflake
  Worksheets also allow them, but they are not documented on the
  website and the SDK also should not advertise them.
* Date and timestamp strings cannot be passed into TIME columns
* Time strings cannot be passed into DATE and TIMESTAMP_* columns
* Some objects from java.time.* package are accepted
    * LocalTime, OffsetTime for TIME column
    * LocalDate, LocalDateTime, OffsetDateTime, ZonedDateTime for
      DATE and TIMESTAMP_* columns
* Time zones conversions are handled correctly
    * For DATE, TIME and TIMESTAMP_NTZ the timestamp is truncated
    * For TIMESTAMP_LTZ, timezone is accounted for before converting
      into the timezone America/Los_Angeles, which is the default
      value of the Snowflake parameter TIMEZONE
    * For TIMESTAMP_TZ, the timezone is preserved
        * America/Los_Angeles is used when timezone is missing
---
 .../streaming/internal/ArrowRowBuffer.java    |   29 +-
 .../internal/DataValidationUtil.java          |  367 ++--
 .../streaming/internal/ArrowBufferTest.java   |    3 +-
 .../internal/DataValidationUtilTest.java      |  413 +++-
 .../streaming/internal/RowBufferTest.java     |   41 +-
 .../streaming/internal/StreamingIngestIT.java |   40 +-
 .../internal/datatypes/DateTimeIT.java        | 1721 +++++++++++++++++
 7 files changed, 2314 insertions(+), 300 deletions(-)
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 83f925ad8..53a7ac10c 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -553,14 +553,17 @@ private float convertRowToArrow(
             }
           case TIMESTAMP_LTZ:
           case TIMESTAMP_NTZ:
+            boolean ignoreTimezone = logicalType == ColumnLogicalType.TIMESTAMP_NTZ;
+
             switch (physicalType) {
               case SB8:
                 {
                   BigIntVector bigIntVector = (BigIntVector) vector;
-                  BigInteger timeInScale =
-                      DataValidationUtil.validateAndParseTime(value, field.getMetadata());
-                  bigIntVector.setSafe(curRowIndex, timeInScale.longValue());
-                  stats.addIntValue(timeInScale);
+                  TimestampWrapper timestampWrapper =
+                      DataValidationUtil.validateAndParseTimestampNtzSb16(
+                          value, getColumnScale(field.getMetadata()), ignoreTimezone);
+                  bigIntVector.setSafe(curRowIndex, timestampWrapper.getTimeInScale().longValue());
+                  stats.addIntValue(timestampWrapper.getTimeInScale());
                   rowBufferSize += 8;
                   break;
                 }
@@ -576,7 +579,7 @@ private float convertRowToArrow(
 
                   TimestampWrapper timestampWrapper =
                       DataValidationUtil.validateAndParseTimestampNtzSb16(
-                          value, field.getMetadata());
+                          value, getColumnScale(field.getMetadata()), ignoreTimezone);
                   epochVector.setSafe(curRowIndex, timestampWrapper.getEpoch());
                   fractionVector.setSafe(curRowIndex, timestampWrapper.getFraction());
                   rowBufferSize += 12;
@@ -600,7 +603,8 @@ private float convertRowToArrow(
                   structVector.setIndexDefined(curRowIndex);
 
                   TimestampWrapper timestampWrapper =
-                      DataValidationUtil.validateAndParseTimestampTz(value, field.getMetadata());
+                      DataValidationUtil.validateAndParseTimestampTz(
+                          value, getColumnScale(field.getMetadata()));
                   epochVector.setSafe(curRowIndex, timestampWrapper.getTimeInScale().longValue());
                   timezoneVector.setSafe(
                       curRowIndex,
@@ -639,7 +643,8 @@ private float convertRowToArrow(
                   structVector.setIndexDefined(curRowIndex);
 
                   TimestampWrapper timestampWrapper =
-                      DataValidationUtil.validateAndParseTimestampTz(value, field.getMetadata());
+                      DataValidationUtil.validateAndParseTimestampTz(
+                          value, getColumnScale(field.getMetadata()));
                   epochVector.setSafe(curRowIndex, timestampWrapper.getEpoch());
                   fractionVector.setSafe(curRowIndex, timestampWrapper.getFraction());
                   timezoneVector.setSafe(
@@ -685,7 +690,8 @@ private float convertRowToArrow(
               case SB4:
                 {
                   BigInteger timeInScale =
-                      DataValidationUtil.validateAndParseTime(value, field.getMetadata());
+                      DataValidationUtil.validateAndParseTime(
+                          value, getColumnScale(field.getMetadata()));
                   stats.addIntValue(timeInScale);
                   ((IntVector) vector).setSafe(curRowIndex, timeInScale.intValue());
                   stats.addIntValue(timeInScale);
@@ -695,7 +701,8 @@ private float convertRowToArrow(
               case SB8:
                 {
                   BigInteger timeInScale =
-                      DataValidationUtil.validateAndParseTime(value, field.getMetadata());
+                      DataValidationUtil.validateAndParseTime(
+                          value, getColumnScale(field.getMetadata()));
                   ((BigIntVector) vector).setSafe(curRowIndex, timeInScale.longValue());
                   stats.addIntValue(timeInScale);
                   rowBufferSize += 8;
@@ -768,6 +775,10 @@ private void insertNull(FieldVector vector, RowBufferStats stats, int curRowInde
     stats.incCurrentNullCount();
   }
 
+  private int getColumnScale(Map<String, String> metadata) {
+    return Integer.parseInt(metadata.get(ArrowRowBuffer.COLUMN_SCALE));
+  }
+
   @Override
   public Flusher<VectorSchemaRoot> createFlusher(Constants.BdecVersion bdecVersion) {
     return new ArrowFlusher(bdecVersion);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index ba01e5681..8f2baa563 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -4,14 +4,24 @@
 
 package net.snowflake.ingest.streaming.internal;
 
+import static net.snowflake.client.jdbc.internal.snowflake.common.core.SnowflakeDateTimeFormat.DATE;
+import static net.snowflake.client.jdbc.internal.snowflake.common.core.SnowflakeDateTimeFormat.TIMESTAMP;
+
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.OffsetDateTime;
+import java.time.OffsetTime;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.TimeZone;
 import java.util.concurrent.TimeUnit;
 import javax.xml.bind.DatatypeConverter;
 import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
@@ -30,9 +40,20 @@ class DataValidationUtil {
   static final BigInteger MAX_BIGINTEGER = BigInteger.valueOf(10).pow(38);
   static final BigInteger MIN_BIGINTEGER =
       BigInteger.valueOf(-1).multiply(BigInteger.valueOf(10).pow(38));
+
+  private static final TimeZone DEFAULT_TIMEZONE =
+      TimeZone.getTimeZone("America/Los_Angeles"); // default value of TIMEZONE system parameter
+  private static final TimeZone GMT = TimeZone.getTimeZone("GMT");
+
   private static final ObjectMapper objectMapper = new ObjectMapper();
-  private static final SnowflakeDateTimeFormat snowflakeDateTimeFormatter =
-      SnowflakeDateTimeFormat.fromSqlFormat("auto");
+
+  /**
+   * Creates a new SnowflakeDateTimeFormat. In order to avoid SnowflakeDateTimeFormat's
+   * synchronization blocks, we create a new instance when needed instead of sharing one instance.
+   */
+  private static SnowflakeDateTimeFormat createDateTimeFormatter() {
+    return SnowflakeDateTimeFormat.fromSqlFormat("auto");
+  }
 
   /**
    * Expects string JSON
@@ -117,72 +138,61 @@ static String validateAndParseObject(Object input) {
   }
 
   /**
-   * Validates and parses input for TIMESTAMP_NTZ Snowflake type
+   * Validates and parses input for TIMESTAMP_NTZ or TIMESTAMP_LTZ Snowflake type. Allowed Java
+   * types:
    *
-   * @param input String date in valid format or seconds past the epoch. Accepts fractional seconds
-   *     with precision up to the column's scale
-   * @param metadata
-   * @return TimestampWrapper with epoch seconds, fractional seconds, and epoch time in the column
-   *     scale
-   */
-  static TimestampWrapper validateAndParseTimestampNtzSb16(
-      Object input, Map<String, String> metadata) {
-    int scale = Integer.parseInt(metadata.get(ArrowRowBuffer.COLUMN_SCALE));
-    return validateAndParseTimestampNtzSb16(input, scale);
-  }
-
-  /**
-   * Validates and parses input for TIMESTAMP_NTZ Snowflake type
+   * <ul>
+   *   <li>String
+   *   <li>LocalDate
+   *   <li>LocalDateTime
+   *   <li>OffsetDateTime
+   *   <li>ZonedDateTime
+   * </ul>
    *
    * @param input String date in valid format or seconds past the epoch. Accepts fractional seconds
    *     with precision up to the column's scale
    * @param scale decimal scale of timestamp 16 byte integer
+   * @param ignoreTimezone Must be true for TIMESTAMP_NTZ
    * @return TimestampWrapper with epoch seconds, fractional seconds, and epoch time in the column
    *     scale
    */
-  static TimestampWrapper validateAndParseTimestampNtzSb16(Object input, int scale) {
-    try {
-      String valueString = getStringValue(input);
-
-      long epoch;
-      int fraction;
-
-      SFTimestamp timestamp = snowflakeDateTimeFormatter.parse(valueString);
-      // The formatter will not correctly parse numbers with decimal values
-      if (timestamp != null) {
-        epoch = timestamp.getSeconds().longValue();
-        fraction = getFractionFromTimestamp(timestamp);
-      } else {
-        /*
-         * Assume the data is of the form "<epoch_seconds>.<fraction of second>". For example "10.5"
-         * is interpreted as meaning 10.5 seconds past the epoch. The following logic breaks out and
-         * stores the epoch seconds and fractional seconds values separately
-         */
-        String[] items = valueString.split("\\.");
-        if (items.length != 2) {
-          throw new SFException(
-              ErrorCode.INVALID_ROW,
-              input.toString(),
-              String.format("Unable to parse timestamp from value.  value=%s", valueString));
-        }
-        epoch = Long.parseLong(items[0]);
-        int l = items.length > 1 ? items[1].length() : 0;
-        /* Fraction is in nanoseconds, but Snowflake will error if the fraction gives
-         * accuracy greater than the scale
-         * */
-        fraction = l == 0 ? 0 : Integer.parseInt(items[1]) * (l < 9 ? Power10.intTable[9 - l] : 1);
-      }
-      if (fraction % Power10.intTable[9 - scale] != 0) {
-        throw new SFException(
-            ErrorCode.INVALID_ROW,
-            input.toString(),
-            String.format(
-                "Row specifies accuracy greater than column scale.  fraction=%d, scale=%d",
-                fraction, scale));
-      }
-      return new TimestampWrapper(epoch, fraction, getTimeInScale(valueString, scale));
-    } catch (NumberFormatException e) {
-      throw new SFException(ErrorCode.INVALID_ROW, input.toString(), e.getMessage());
+  static TimestampWrapper validateAndParseTimestampNtzSb16(
+      Object input, int scale, boolean ignoreTimezone) {
+    String valueString;
+    if (input instanceof String) valueString = (String) input;
+    else if (input instanceof LocalDate) valueString = input.toString();
+    else if (input instanceof LocalDateTime) valueString = input.toString();
+    else if (input instanceof ZonedDateTime)
+      valueString =
+          ignoreTimezone
+              ? ((ZonedDateTime) input).toLocalDateTime().toString()
+              : DateTimeFormatter.ISO_DATE_TIME.format(((ZonedDateTime) input).toOffsetDateTime());
+    else if (input instanceof OffsetDateTime)
+      valueString =
+          ignoreTimezone
+              ? ((OffsetDateTime) input).toLocalDateTime().toString()
+              : DateTimeFormatter.ISO_DATE_TIME.format((OffsetDateTime) input);
+    else
+      throw typeNotAllowedException(
+          input.getClass(),
+          "TIMESTAMP",
+          new String[] {"String", "LocalDate", "LocalDateTime", "ZonedDateTime", "OffsetDateTime"});
+
+    SnowflakeDateTimeFormat snowflakeDateTimeFormatter = createDateTimeFormatter();
+    TimeZone effectiveTimeZone = ignoreTimezone ? GMT : DEFAULT_TIMEZONE;
+    SFTimestamp timestamp =
+        snowflakeDateTimeFormatter.parse(
+            valueString, effectiveTimeZone, 0, DATE | TIMESTAMP, ignoreTimezone, null);
+    if (timestamp != null) {
+      long epoch = timestamp.getSeconds().longValue();
+      int fraction = getFractionFromTimestamp(timestamp) / Power10.intTable[9 - scale];
+      BigInteger timeInScale =
+          BigInteger.valueOf(epoch)
+              .multiply(Power10.sb16Table[scale])
+              .add(BigInteger.valueOf(fraction));
+      return new TimestampWrapper(epoch, fraction, timeInScale);
+    } else {
+      throw valueFormatNotAllowedException(input.toString(), "TIMESTAMP");
     }
   }
 
@@ -200,20 +210,15 @@ private static int getFractionFromTimestamp(SFTimestamp input) {
   }
 
   /**
-   * Validates and parses input for TIMESTAMP_TZ Snowflake type
+   * Validates and parses input for TIMESTAMP_TZ Snowflake type. Allowed Java types:
    *
-   * @param input TIMESTAMP_TZ in "2021-01-01 01:00:00 +0100" format
-   * @param metadata Column metadata containing scale
-   * @return TimestampWrapper with epoch seconds, fractional seconds, and epoch time in the column
-   *     scale
-   */
-  static TimestampWrapper validateAndParseTimestampTz(Object input, Map<String, String> metadata) {
-    int scale = Integer.parseInt(metadata.get(ArrowRowBuffer.COLUMN_SCALE));
-    return validateAndParseTimestampTz(input, scale);
-  }
-
-  /**
-   * Validates and parses input for TIMESTAMP_TZ Snowflake type
+   * <ul>
+   *   <li>String
+   *   <li>LocalDate
+   *   <li>LocalDateTime
+   *   <li>OffsetDateTime
+   *   <li>ZonedDateTime
+   * </ul>
    *
    * @param input TIMESTAMP_TZ in "2021-01-01 01:00:00 +0100" format
    * @param scale decimal scale of timestamp 16 byte integer
@@ -221,40 +226,37 @@ static TimestampWrapper validateAndParseTimestampTz(Object input, Map<String, St
    *     scale
    */
   static TimestampWrapper validateAndParseTimestampTz(Object input, int scale) {
-    try {
-      if (input instanceof String) {
-        String stringInput = (String) input;
-        SFTimestamp timestamp = snowflakeDateTimeFormatter.parse(stringInput);
-        if (timestamp == null) {
-          throw new SFException(
-              ErrorCode.INVALID_ROW, input, "Cannot be converted to a timestamp_tz value");
-        }
-
-        int fraction = getFractionFromTimestamp(timestamp);
 
-        BigDecimal epochTimeAtSecondsScale =
-            timestamp.getNanosSinceEpoch().divide(BigDecimal.valueOf(Power10.intTable[9]));
-        if (fraction % Power10.intTable[9 - scale] != 0) {
-          throw new SFException(
-              ErrorCode.INVALID_ROW,
-              input.toString(),
-              String.format(
-                  "Row specifies accuracy greater than column scale.  fraction=%d, scale=%d",
-                  fraction, scale));
-        }
-        TimestampWrapper wrapper =
-            new TimestampWrapper(
-                timestamp.getSeconds().longValue(),
-                fraction,
-                getTimeInScale(getStringValue(epochTimeAtSecondsScale), scale),
-                timestamp);
-        return wrapper;
-      } else {
-        throw new SFException(
-            ErrorCode.INVALID_ROW, input, "Cannot be converted to a timestamp_tz value");
-      }
-    } catch (NumberFormatException e) {
-      throw new SFException(ErrorCode.INVALID_ROW, input.toString(), e.getMessage());
+    String stringInput;
+    if (input instanceof String) stringInput = (String) input;
+    else if (input instanceof LocalDate) stringInput = input.toString();
+    else if (input instanceof LocalDateTime) stringInput = input.toString();
+    else if (input instanceof ZonedDateTime)
+      stringInput =
+          DateTimeFormatter.ISO_DATE_TIME.format(((ZonedDateTime) input).toOffsetDateTime());
+    else if (input instanceof OffsetDateTime)
+      stringInput = DateTimeFormatter.ISO_DATE_TIME.format((OffsetDateTime) input);
+    else
+      throw typeNotAllowedException(
+          input.getClass(),
+          "TIMESTAMP",
+          new String[] {"String", "LocalDate", "LocalDateTime", "ZonedDateTime", "OffsetDateTime"});
+
+    SnowflakeDateTimeFormat snowflakeDateTimeFormatter = createDateTimeFormatter();
+
+    SFTimestamp timestamp =
+        snowflakeDateTimeFormatter.parse(
+            stringInput, DEFAULT_TIMEZONE, 0, DATE | TIMESTAMP, false, null);
+    if (timestamp != null) {
+      long epoch = timestamp.getSeconds().longValue();
+      int fraction = getFractionFromTimestamp(timestamp) / Power10.intTable[9 - scale];
+      return new TimestampWrapper(
+          epoch,
+          fraction,
+          BigInteger.valueOf(epoch * Power10.intTable[scale] + fraction),
+          timestamp);
+    } else {
+      throw valueFormatNotAllowedException(input.toString(), "TIMESTAMP");
     }
   }
 
@@ -536,28 +538,40 @@ static long validateAndParseLong(Object input) {
   }
 
   /**
-   * Expects input in epoch days or "YYYY-MM-DD" format. Uses @see {@link
-   * #validateAndParseInteger(Object)} to validate and parse input as integer
+   * Returns the number of days between the epoch and the passed date. Allowed Java types:
    *
-   * @param input epoch days or "YYYY-MM-DD" date format
+   * <ul>
+   *   <li>String
+   *   <li>LocalDate
+   *   <li>LocalDateTime
+   *   <li>OffsetDateTime
+   *   <li>ZonedDateTime
+   * </ul>
    */
   static int validateAndParseDate(Object input) {
-    try {
-      return validateAndParseInteger(input);
-    } catch (SFException e) {
-      // Try parsing the time from a String
-      Optional<SFTimestamp> timestamp =
-          Optional.ofNullable(snowflakeDateTimeFormatter.parse(input.toString()));
-      return timestamp
-          .map(
-              t ->
-                  // Adjust nanoseconds past the epoch to match the column scale value
-                  (int) TimeUnit.MILLISECONDS.toDays(SFDate.fromTimestamp(t).getTime()))
-          .orElseThrow(
-              () ->
-                  new SFException(
-                      ErrorCode.INVALID_ROW, input, "Cannot be converted to a Date value"));
+    String inputString;
+    if (input instanceof String) {
+      inputString = (String) input;
+    } else if (input instanceof LocalDate) {
+      inputString = input.toString();
+    } else if (input instanceof LocalDateTime) {
+      inputString = ((LocalDateTime) input).toLocalDate().toString();
+    } else if (input instanceof ZonedDateTime) {
+      inputString = ((ZonedDateTime) input).toLocalDate().toString();
+    } else if (input instanceof OffsetDateTime) {
+      inputString = ((OffsetDateTime) input).toLocalDate().toString();
+    } else {
+      throw typeNotAllowedException(
+          input.getClass(),
+          "DATE",
+          new String[] {"String", "LocalDate", "LocalDateTime", "ZonedDateTime", "OffsetDateTime"});
     }
+
+    SFTimestamp timestamp =
+        createDateTimeFormatter().parse(inputString, GMT, 0, DATE | TIMESTAMP, true, null);
+    if (timestamp == null) throw valueFormatNotAllowedException(input, "DATE");
+
+    return (int) TimeUnit.MILLISECONDS.toDays(SFDate.fromTimestamp(timestamp).getTime());
   }
 
   static byte[] validateAndParseBinary(Object input, Optional<Integer> maxLengthOptional) {
@@ -583,41 +597,39 @@ static byte[] validateAndParseBinary(Object input, Optional<Integer> maxLengthOp
   }
 
   /**
-   * @param input Seconds past the epoch. or String Time representation
-   * @param metadata
-   */
-  static BigInteger validateAndParseTime(Object input, Map<String, String> metadata) {
-    int scale = Integer.parseInt(metadata.get(ArrowRowBuffer.COLUMN_SCALE));
-    return getTimeInScale(getStringValue(input), scale);
-  }
-
-  /**
-   * Returns the time past the epoch in the given scale.
+   * Returns the number of units since 00:00, depending on the scale (scale=0: seconds, scale=3:
+   * milliseconds, scale=9: nanoseconds. Allowed Java types:
    *
-   * @param value
-   * @param scale Value between 0 and 9. For seconds scale = 0, for milliseconds scale = 3, for
-   *     nanoseconds scale = 9
+   * <ul>
+   *   <li>String
+   *   <li>LocalTime
+   *   <li>OffsetTime
+   * </ul>
    */
-  static BigInteger getTimeInScale(String value, int scale) {
-    try {
-      BigDecimal decVal = new BigDecimal(value);
-      BigDecimal epochScale = decVal.multiply(BigDecimal.valueOf(Power10.intTable[scale]));
-      return epochScale.toBigInteger();
-    } catch (NumberFormatException e) {
-      // Try parsing the time from a String
-      Optional<SFTimestamp> timestamp =
-          Optional.ofNullable(snowflakeDateTimeFormatter.parse(value));
+  static BigInteger validateAndParseTime(Object input, int scale) {
+    String stringInput;
+    if (input instanceof String) {
+      stringInput = (String) input;
+    } else if (input instanceof LocalTime) {
+      stringInput = input.toString();
+    } else if (input instanceof OffsetTime) {
+      stringInput = ((OffsetTime) input).toLocalTime().toString();
+    } else {
+      throw typeNotAllowedException(
+          input.getClass(), "TIME", new String[] {"String", "LocalTime", "OffsetTime"});
+    }
+
+    SFTimestamp timestamp =
+        createDateTimeFormatter()
+            .parse(stringInput, GMT, 0, SnowflakeDateTimeFormat.TIME, true, null);
+    if (timestamp == null) {
+      throw valueFormatNotAllowedException(input, "TIME");
+    } else {
       return timestamp
-          .map(
-              t ->
-                  // Adjust nanoseconds past the epoch to match the column scale value
-                  t.getNanosSinceEpoch()
-                      .divide(BigDecimal.valueOf(Power10.intTable[9 - scale]))
-                      .toBigInteger())
-          .orElseThrow(
-              () ->
-                  new SFException(
-                      ErrorCode.INVALID_ROW, value, "Cannot be converted to a time value"));
+          .getNanosSinceEpoch()
+          .divide(BigDecimal.valueOf(Power10.intTable[9 - scale]))
+          .toBigInteger()
+          .mod(BigInteger.valueOf(24L * 60 * 60 * Power10.intTable[scale]));
     }
   }
 
@@ -661,8 +673,15 @@ static double validateAndParseReal(Object input) {
   /**
    * Validate and parse input to integer output, 1=true, 0=false. String values converted to boolean
    * according to https://docs.snowflake.com/en/sql-reference/functions/to_boolean.html#usage-notes
+   * Allowed Java types:
    *
-   * @param input
+   * <ul>
+   *   <li>boolean
+   *   <li>String
+   *   <li>java.lang.Number
+   * </ul>
+   *
+   * @param input Input to be converted
    * @return 1 or 0 where 1=true, 0=false
    */
   static int validateAndParseBoolean(Object input) {
@@ -674,7 +693,8 @@ static int validateAndParseBoolean(Object input) {
       return convertStringToBoolean((String) input) ? 1 : 0;
     }
 
-    throw cannotConvertException(input, "boolean");
+    throw typeNotAllowedException(
+        input.getClass(), "BOOLEAN", new String[] {"boolean", "Number", "String"});
   }
 
   static Set<String> allowedBooleanStringsLowerCased =
@@ -683,7 +703,7 @@ static int validateAndParseBoolean(Object input) {
   private static boolean convertStringToBoolean(String value) {
     String lowerCasedValue = value.toLowerCase();
     if (!allowedBooleanStringsLowerCased.contains(lowerCasedValue)) {
-      throw cannotConvertException(value, "boolean");
+      throw valueFormatNotAllowedException(value, "BOOLEAN");
     }
     return "1".equals(lowerCasedValue)
         || "yes".equals(lowerCasedValue)
@@ -694,16 +714,35 @@ private static boolean convertStringToBoolean(String value) {
   }
 
   /**
-   * Create exception when value cannot be ingested into column of a specific type.
+   * Create exception that a Java type cannot be ingested into a specific Snowflake column type
+   *
+   * @param javaType Java type failing the validation
+   * @param snowflakeType Target Snowflake column type
+   * @param allowedJavaTypes Java types supported for the Java type
+   */
+  private static SFException typeNotAllowedException(
+      Class<?> javaType, String snowflakeType, String[] allowedJavaTypes) {
+    return new SFException(
+        ErrorCode.INVALID_ROW,
+        String.format(
+            "Object of type %s cannot be ingested into Snowflake column of type %s",
+            javaType.getName(), snowflakeType),
+        String.format(
+            String.format("Allowed Java types: %s", String.join(", ", allowedJavaTypes))));
+  }
+
+  /**
+   * Create exception when the Java type is correct, but the value is invalid (e.g. boolean cannot
+   * be parsed from a string)
    *
    * @param value Invalid value causing the exception
-   * @param type Snowflake column type
+   * @param snowflakeType Snowflake column type
    */
-  private static SFException cannotConvertException(Object value, String type) {
+  private static SFException valueFormatNotAllowedException(Object value, String snowflakeType) {
     return new SFException(
         ErrorCode.INVALID_ROW,
         sanitizeValueForExceptionMessage(value),
-        String.format("Value cannot be converted to %s", type));
+        String.format("Value cannot be ingested into Snowflake column %s", snowflakeType));
   }
 
   /**
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
index e8228b9f3..c07966c68 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
@@ -2,7 +2,6 @@
 
 import static net.snowflake.ingest.streaming.internal.ArrowRowBuffer.DECIMAL_BIT_WIDTH;
 
-import java.math.BigDecimal;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -432,7 +431,7 @@ private void testArrowE2ETimestampLTZHelper(OpenChannelRequest.OnErrorOption onE
 
     Map<String, Object> row = new HashMap<>();
     row.put("COLTIMESTAMPLTZ_SB8", "1621899220");
-    row.put("COLTIMESTAMPLTZ_SB16", new BigDecimal("1621899220.123456789"));
+    row.put("COLTIMESTAMPLTZ_SB16", "1621899220123456789");
 
     InsertValidationResponse response =
         innerBuffer.insertRows(Collections.singletonList(row), null);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 99e0aae3e..a52008cf9 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -2,6 +2,10 @@
 
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.MAX_BIGINTEGER;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseBoolean;
+import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseDate;
+import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseTime;
+import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseTimestampNtzSb16;
+import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseTimestampTz;
 import static org.junit.Assert.assertEquals;
 
 import com.fasterxml.jackson.databind.JsonNode;
@@ -9,7 +13,14 @@
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.OffsetDateTime;
+import java.time.OffsetTime;
+import java.time.ZonedDateTime;
 import java.util.Arrays;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -31,17 +42,24 @@ private void expectError(ErrorCode expectedErrorCode, Function func, Object args
     expectError(expectedErrorCode, () -> func.apply(args));
   }
 
-  private void expectError(ErrorCode expectedErrorCode, Runnable action) {
+  private void expectErrorCodeAndMessage(
+      ErrorCode expectedErrorCode, String expectedExceptionMessage, Runnable action) {
     try {
       action.run();
       Assert.fail("Expected Exception");
     } catch (SFException e) {
       assertEquals(expectedErrorCode.getMessageCode(), e.getVendorCode());
+      if (expectedExceptionMessage != null)
+        Assert.assertEquals(expectedExceptionMessage, e.getMessage());
     } catch (Exception e) {
       Assert.fail("Invalid error through");
     }
   }
 
+  private void expectError(ErrorCode expectedErrorCode, Runnable action) {
+    expectErrorCodeAndMessage(expectedErrorCode, null, action);
+  }
+
   @Test
   public void testValidateAndParseShort() {
     short e = 12;
@@ -58,96 +76,205 @@ public void testValidateAndParseShort() {
 
   @Test
   public void testValidateAndParseTime() {
-    Map<String, String> metadata = new HashMap<>();
-    metadata.put(ArrowRowBuffer.COLUMN_SCALE, "0");
-    Assert.assertEquals(
-        new BigInteger("1595289600"),
-        DataValidationUtil.validateAndParseTime("1595289600", metadata));
-
-    metadata.put(ArrowRowBuffer.COLUMN_SCALE, "0");
-    Assert.assertEquals(
-        new BigInteger("1595289600"),
-        DataValidationUtil.validateAndParseTime("2020-07-21", metadata));
-
-    Assert.assertEquals(
-        new BigInteger("1595374380"),
-        DataValidationUtil.validateAndParseTime("2020-07-21 23:33:00", metadata));
-
-    metadata.put(ArrowRowBuffer.COLUMN_SCALE, "3");
-    Assert.assertEquals(
-        new BigInteger("1595289600000"),
-        DataValidationUtil.validateAndParseTime("1595289600", metadata));
-
-    Assert.assertEquals(
-        new BigInteger("1595289600000"),
-        DataValidationUtil.validateAndParseTime("2020-07-21", metadata));
+    assertEquals(5L, validateAndParseTime("00:00:05", 0).longValueExact());
+    assertEquals(5000L, validateAndParseTime("00:00:05", 3).longValueExact());
+    assertEquals(5000L, validateAndParseTime("00:00:05.000", 3).longValueExact());
+    assertEquals(5123L, validateAndParseTime("00:00:05.123", 3).longValueExact());
+    assertEquals(5123L, validateAndParseTime("00:00:05.123456", 3).longValueExact());
+    assertEquals(5123456789L, validateAndParseTime("00:00:05.123456789", 9).longValueExact());
+
+    assertEquals(72L, validateAndParseTime("72", 0).longValueExact());
+    assertEquals(72000L, validateAndParseTime("72", 3).longValueExact());
+
+    // Timestamps are rejected
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2/18/2008 02:36:48", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57:01", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57:01 +07:00", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57:01 +0700", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57:01-07", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57:01-07:00", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57:01.123456", 9));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime("2013-04-28 20:57:01.123456789 +07:00", 9));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime("2013-04-28 20:57:01.123456789 +0700", 9));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57:01.123456789+07", 9));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime("2013-04-28 20:57:01.123456789+07:00", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57+07:00", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28T20", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28T20:57", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28T20:57:01", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28T20:57:01-07:00", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28T20:57:01.123456", 9));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime("2013-04-28T20:57:01.123456789+07:00", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28T20:57+07:00", 9));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime("Mon Jul 08 18:09:51 +0000 2013", 9));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime("Thu, 21 Dec 2000 04:01:07 PM", 9));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime("Thu, 21 Dec 2000 04:01:07 PM +0200", 9));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime("Thu, 21 Dec 2000 04:01:07.123456789 PM", 9));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime("Thu, 21 Dec 2000 04:01:07.123456789 PM +0200", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("Thu, 21 Dec 2000 16:01:07", 9));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime("Thu, 21 Dec 2000 16:01:07 +0200", 9));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime("Thu, 21 Dec 2000 16:01:07.123456789", 9));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime("Thu, 21 Dec 2000 16:01:07.123456789 +0200", 9));
 
-    Assert.assertEquals(
-        new BigInteger("1595374380000"),
-        DataValidationUtil.validateAndParseTime("2020-07-21 23:33:00", metadata));
+    // Dates are rejected
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("17-DEC-1980", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("12/17/1980", 9));
 
-    Assert.assertEquals(
-        new BigInteger("1595374380123"),
-        DataValidationUtil.validateAndParseTime("2020-07-21 23:33:00.123", metadata));
+    // Test forbidden values
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(LocalDate.now(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(LocalDateTime.now(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(OffsetDateTime.now(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(ZonedDateTime.now(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(new Date(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(1.5f, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(1.5, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("1.5", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("1.0", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(new Object(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(false, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("foo", 3));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime(java.sql.Time.valueOf("20:57:00"), 3));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime(java.sql.Date.valueOf("2010-11-03"), 3));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime(java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(BigInteger.ZERO, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(BigDecimal.ZERO, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime('c', 3));
   }
 
   @Test
-  public void testValidateAndPareTimestampNtzSb16() {
-    Map<String, String> metadata = new HashMap<>();
-    metadata.put(ArrowRowBuffer.COLUMN_SCALE, "3");
-    Assert.assertEquals(
-        new TimestampWrapper(1, 123000000, BigInteger.valueOf(1123)),
-        DataValidationUtil.validateAndParseTimestampNtzSb16("1.123", metadata));
-    metadata.put(ArrowRowBuffer.COLUMN_SCALE, "9");
-    Assert.assertEquals(
-        new TimestampWrapper(1, 123, BigInteger.valueOf(1000000123)),
-        DataValidationUtil.validateAndParseTimestampNtzSb16("1.000000123", metadata));
-
-    Assert.assertEquals(
+  public void testValidateAndParseTimestampNtzSb16() {
+    assertEquals(
         new TimestampWrapper(1609462800, 123000000, new BigInteger("1609462800123000000")),
-        DataValidationUtil.validateAndParseTimestampNtzSb16("2021-01-01 01:00:00.123", metadata));
+        DataValidationUtil.validateAndParseTimestampNtzSb16("2021-01-01 01:00:00.123", 9, true));
 
-    // Expect errors
-    try {
-      DataValidationUtil.validateAndParseTimestampNtzSb16("honk", metadata);
-      Assert.fail("Expected Exception");
-    } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
-    }
-    metadata.put(ArrowRowBuffer.COLUMN_SCALE, "1");
-    try {
-      DataValidationUtil.validateAndParseTimestampNtzSb16("1.23", metadata);
-      Assert.fail("Expected Exception");
-    } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
-    }
+    // Time formats are not supported
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestampNtzSb16("20:57:01.123456789+07:00", 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestampNtzSb16("20:57:01.123456789", 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("20:57:01", 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("20:57", 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestampNtzSb16("07:57:01.123456789 AM", 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("04:01:07 AM", 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("04:01 AM", 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("04:01 PM", 3, false));
+
+    // Test forbidden values
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(LocalTime.now(), 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(OffsetTime.now(), 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(new Date(), 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(1.5f, 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(1.5, 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("1.5", 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("1.0", 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(new Object(), 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(false, 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("", 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("foo", 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestampNtzSb16(java.sql.Time.valueOf("20:57:00"), 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestampNtzSb16(java.sql.Date.valueOf("2010-11-03"), 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () ->
+            validateAndParseTimestampNtzSb16(
+                java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(BigInteger.ZERO, 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(BigDecimal.ZERO, 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16('c', 3, false));
   }
 
   @Test
   public void testValidateAndPareTimestampTz() {
-    Map<String, String> metadata = new HashMap<>();
-    metadata.put(ArrowRowBuffer.COLUMN_SCALE, "3");
     TimestampWrapper result =
-        DataValidationUtil.validateAndParseTimestampTz("2021-01-01 01:00:00.123 +0100", metadata);
-    Assert.assertEquals(1609459200, result.getEpoch());
-    Assert.assertEquals(123000000, result.getFraction());
-    Assert.assertEquals(Optional.of(3600000), result.getTimezoneOffset());
-    Assert.assertEquals(Optional.of(1500), result.getTimeZoneIndex());
+        DataValidationUtil.validateAndParseTimestampTz("2021-01-01 01:00:00.123 +0100", 4);
+    assertEquals(1609459200, result.getEpoch());
+    assertEquals(1230, result.getFraction());
+    assertEquals(Optional.of(3600000), result.getTimezoneOffset());
+    assertEquals(Optional.of(1500), result.getTimeZoneIndex());
 
-    // Expect errors
-    try {
-      DataValidationUtil.validateAndParseTimestampNtzSb16("honk", metadata);
-      Assert.fail("Expected Exception");
-    } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
-    }
-    metadata.put(ArrowRowBuffer.COLUMN_SCALE, "1");
-    try {
-      DataValidationUtil.validateAndParseTimestampTz("2021-01-01 01:00:00.123 +0100", metadata);
-      Assert.fail("Expected Exception");
-    } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
-    }
+    // Time formats are not supported
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("20:57:01.123456789+07:00", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("20:57:01.123456789", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("20:57:01", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("20:57", 3));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("07:57:01.123456789 AM", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("04:01:07 AM", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("04:01 AM", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("04:01 PM", 3));
+
+    // Test forbidden values
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(LocalTime.now(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(OffsetTime.now(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(new Date(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(1.5f, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(1.5, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("1.5", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("1.0", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(new Object(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(false, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("foo", 3));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestampTz(java.sql.Time.valueOf("20:57:00"), 3));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestampTz(java.sql.Date.valueOf("2010-11-03"), 3));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestampTz(java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(BigInteger.ZERO, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(BigDecimal.ZERO, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz('c', 3));
   }
 
   @Test
@@ -452,11 +579,39 @@ public void testValidateAndParseLong() {
 
   @Test
   public void testValidateAndParseDate() {
-    Assert.assertEquals(-923, DataValidationUtil.validateAndParseDate("1967-06-23"));
-    Assert.assertEquals(-923, DataValidationUtil.validateAndParseDate("1967-06-23 01:01:01"));
-    Assert.assertEquals(18464, DataValidationUtil.validateAndParseDate("2020-07-21"));
-    Assert.assertEquals(18464, DataValidationUtil.validateAndParseDate("2020-07-21 23:31:00"));
-    Assert.assertEquals(12341, DataValidationUtil.validateAndParseDate(12341));
+    assertEquals(-923, validateAndParseDate("1967-06-23"));
+    assertEquals(-923, validateAndParseDate("1967-06-23 01:01:01"));
+    assertEquals(18464, validateAndParseDate("2020-07-21"));
+    assertEquals(18464, validateAndParseDate("2020-07-21 23:31:00"));
+
+    // Time formats are not supported
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("20:57:01.123456789+07:00"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("20:57:01.123456789"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("20:57:01"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("20:57"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("07:57:01.123456789 AM"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("04:01:07 AM"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("04:01 AM"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("04:01 PM"));
+
+    // Test forbidden values
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, new Object());
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, LocalTime.now());
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, OffsetTime.now());
+    expectError(
+        ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, new java.util.Date());
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, false);
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, "");
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, "foo");
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, "1.0");
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, 'c');
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, 1);
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, 1L);
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, 1.25);
+    expectError(
+        ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, BigInteger.valueOf(1));
+    expectError(
+        ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, BigDecimal.valueOf(1.25));
   }
 
   @Test
@@ -470,13 +625,6 @@ public void testGetStringValue() throws Exception {
     Assert.assertEquals("123", DataValidationUtil.getStringValue(123l));
   }
 
-  @Test
-  public void testGetTimestampInScale() throws Exception {
-    Assert.assertEquals(
-        new BigInteger("123"), DataValidationUtil.getTimeInScale("123.000000000", 0));
-    Assert.assertEquals(new BigInteger("12301"), DataValidationUtil.getTimeInScale("123.01", 2));
-  }
-
   @Test
   public void testValidateAndParseBinary() {
     Assert.assertArrayEquals(
@@ -551,4 +699,89 @@ public void testValidateAndParseBoolean() {
     expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBoolean, "foobar");
     expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBoolean, "");
   }
+
+  /**
+   * Tests that exception message are constructed correctly when ingesting forbidden Java type, as
+   * well a value of an allowed type, but in invalid format
+   */
+  @Test
+  public void testExceptionMessages() {
+    // BOOLEAN
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
+            + " be ingested into Snowflake column of type BOOLEAN. Allowed Java types: boolean,"
+            + " Number, String",
+        () -> validateAndParseBoolean(new Object()));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
+            + " Snowflake column BOOLEAN",
+        () -> validateAndParseBoolean("abc"));
+
+    // TIME
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
+            + " be ingested into Snowflake column of type TIME. Allowed Java types: String,"
+            + " LocalTime, OffsetTime",
+        () -> validateAndParseTime(new Object(), 10));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
+            + " Snowflake column TIME",
+        () -> validateAndParseTime("abc", 10));
+
+    // DATE
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
+            + " be ingested into Snowflake column of type DATE. Allowed Java types: String,"
+            + " LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
+        () -> validateAndParseDate(new Object()));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
+            + " Snowflake column DATE",
+        () -> validateAndParseDate("abc"));
+
+    // TIMESTAMP_NTZ
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
+            + " be ingested into Snowflake column of type TIMESTAMP. Allowed Java types: String,"
+            + " LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
+        () -> validateAndParseTimestampNtzSb16(new Object(), 3, true));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
+            + " Snowflake column TIMESTAMP",
+        () -> validateAndParseTimestampNtzSb16("abc", 3, true));
+
+    // TIMESTAMP_LTZ
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
+            + " be ingested into Snowflake column of type TIMESTAMP. Allowed Java types: String,"
+            + " LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
+        () -> validateAndParseTimestampNtzSb16(new Object(), 3, false));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
+            + " Snowflake column TIMESTAMP",
+        () -> validateAndParseTimestampNtzSb16("abc", 3, false));
+
+    // TIMESTAMP_TZ
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
+            + " be ingested into Snowflake column of type TIMESTAMP. Allowed Java types: String,"
+            + " LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
+        () -> validateAndParseTimestampTz(new Object(), 3));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
+            + " Snowflake column TIMESTAMP",
+        () -> validateAndParseTimestampTz("abc", 3));
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 4f2e12b86..75e053d10 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -683,13 +683,13 @@ private void testStatsE2ETimestampHelper(OpenChannelRequest.OnErrorOption onErro
 
     Map<String, Object> row1 = new HashMap<>();
     row1.put("COLTIMESTAMPLTZ_SB8", "1621899220");
-    row1.put("COLTIMESTAMPLTZ_SB16", "1621899220.123456789");
-    row1.put("COLTIMESTAMPLTZ_SB16_SCALE6", "1621899220.123456");
+    row1.put("COLTIMESTAMPLTZ_SB16", "1621899220123456789");
+    row1.put("COLTIMESTAMPLTZ_SB16_SCALE6", "1621899220123456");
 
     Map<String, Object> row2 = new HashMap<>();
     row2.put("COLTIMESTAMPLTZ_SB8", "1621899221");
-    row2.put("COLTIMESTAMPLTZ_SB16", "1621899220.12345679");
-    row2.put("COLTIMESTAMPLTZ_SB16_SCALE6", "1621899220.123457");
+    row2.put("COLTIMESTAMPLTZ_SB16", "1621899220223456789");
+    row2.put("COLTIMESTAMPLTZ_SB16_SCALE6", "1621899220123457");
 
     Map<String, Object> row3 = new HashMap<>();
     row3.put("COLTIMESTAMPLTZ_SB8", null);
@@ -713,7 +713,7 @@ private void testStatsE2ETimestampHelper(OpenChannelRequest.OnErrorOption onErro
         new BigInteger("1621899220123456789"),
         result.getColumnEps().get("COLTIMESTAMPLTZ_SB16").getCurrentMinIntValue());
     Assert.assertEquals(
-        new BigInteger("1621899220123456790"),
+        new BigInteger("1621899220223456789"),
         result.getColumnEps().get("COLTIMESTAMPLTZ_SB16").getCurrentMaxIntValue());
 
     Assert.assertEquals(
@@ -748,10 +748,10 @@ private void testE2EDateHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
     innerBuffer.setupSchema(Collections.singletonList(colDate));
 
     Map<String, Object> row1 = new HashMap<>();
-    row1.put("COLDATE", "18772");
+    row1.put("COLDATE", String.valueOf(18772 * 24 * 60 * 60 * 1000L + 1));
 
     Map<String, Object> row2 = new HashMap<>();
-    row2.put("COLDATE", "18773");
+    row2.put("COLDATE", String.valueOf(18773 * 24 * 60 * 60 * 1000L + 1));
 
     Map<String, Object> row3 = new HashMap<>();
     row3.put("COLDATE", null);
@@ -803,12 +803,12 @@ private void testE2ETimeHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
     innerBuffer.setupSchema(Arrays.asList(colTimeSB4, colTimeSB8));
 
     Map<String, Object> row1 = new HashMap<>();
-    row1.put("COLTIMESB4", "43200");
-    row1.put("COLTIMESB8", "44200.123");
+    row1.put("COLTIMESB4", "10:00:00");
+    row1.put("COLTIMESB8", "10:00:00.123");
 
     Map<String, Object> row2 = new HashMap<>();
-    row2.put("COLTIMESB4", "43260");
-    row2.put("COLTIMESB8", "44201");
+    row2.put("COLTIMESB4", "11:15:00.000");
+    row2.put("COLTIMESB8", "11:15:00.456");
 
     Map<String, Object> row3 = new HashMap<>();
     row3.put("COLTIMESB4", null);
@@ -819,12 +819,13 @@ private void testE2ETimeHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
     Assert.assertFalse(response.hasErrors());
 
     // Check data was inserted into the buffer correctly
-    Assert.assertEquals(43200, innerBuffer.getVectorValueAt("COLTIMESB4", 0));
-    Assert.assertEquals(43260, innerBuffer.getVectorValueAt("COLTIMESB4", 1));
+    Assert.assertEquals(10 * 60 * 60, innerBuffer.getVectorValueAt("COLTIMESB4", 0));
+    Assert.assertEquals(11 * 60 * 60 + 15 * 60, innerBuffer.getVectorValueAt("COLTIMESB4", 1));
     Assert.assertNull(innerBuffer.getVectorValueAt("COLTIMESB4", 2));
 
-    Assert.assertEquals(44200123L, innerBuffer.getVectorValueAt("COLTIMESB8", 0));
-    Assert.assertEquals(44201000L, innerBuffer.getVectorValueAt("COLTIMESB8", 1));
+    Assert.assertEquals(10 * 60 * 60 * 1000L + 123, innerBuffer.getVectorValueAt("COLTIMESB8", 0));
+    Assert.assertEquals(
+        11 * 60 * 60 * 1000L + 15 * 60 * 1000 + 456, innerBuffer.getVectorValueAt("COLTIMESB8", 1));
     Assert.assertNull(innerBuffer.getVectorValueAt("COLTIMESB8", 2));
 
     // Check stats generation
@@ -832,16 +833,18 @@ private void testE2ETimeHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
     Assert.assertEquals(3, result.getRowCount());
 
     Assert.assertEquals(
-        BigInteger.valueOf(43200), result.getColumnEps().get("COLTIMESB4").getCurrentMinIntValue());
+        BigInteger.valueOf(10 * 60 * 60),
+        result.getColumnEps().get("COLTIMESB4").getCurrentMinIntValue());
     Assert.assertEquals(
-        BigInteger.valueOf(43260), result.getColumnEps().get("COLTIMESB4").getCurrentMaxIntValue());
+        BigInteger.valueOf(11 * 60 * 60 + 15 * 60),
+        result.getColumnEps().get("COLTIMESB4").getCurrentMaxIntValue());
     Assert.assertEquals(1, result.getColumnEps().get("COLTIMESB4").getCurrentNullCount());
 
     Assert.assertEquals(
-        BigInteger.valueOf(44200123),
+        BigInteger.valueOf(10 * 60 * 60 * 1000L + 123),
         result.getColumnEps().get("COLTIMESB8").getCurrentMinIntValue());
     Assert.assertEquals(
-        BigInteger.valueOf(44201000),
+        BigInteger.valueOf(11 * 60 * 60 * 1000L + 15 * 60 * 1000 + 456),
         result.getColumnEps().get("COLTIMESB8").getCurrentMaxIntValue());
     Assert.assertEquals(1, result.getColumnEps().get("COLTIMESB8").getCurrentNullCount());
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 7642a591e..6b49fbf29 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -396,22 +396,22 @@ public void testTimeColumnIngest() throws Exception {
     row.put("ttzbig", "2021-01-01 09:00:00.12345678 -0300");
     row.put("tsmall", "01:00:00.123");
     row.put("tbig", "09:00:00.12345678");
-    row.put("tntzsmall", "1609462800.123");
-    row.put("tntzbig", "1609462800.12345");
+    row.put("tntzsmall", "1609462800123");
+    row.put("tntzbig", "1609462800123450000");
     verifyInsertValidationResponse(channel1.insertRow(row, null));
     row.put("ttzsmall", "2021-01-01 10:00:00.123 +0700");
     row.put("ttzbig", "2021-01-01 19:00:00.12345678 -0300");
     row.put("tsmall", "02:00:00.123");
     row.put("tbig", "10:00:00.12345678");
-    row.put("tntzsmall", "1709462800.123");
-    row.put("tntzbig", "1709462800.12345");
+    row.put("tntzsmall", "1709462800123");
+    row.put("tntzbig", "170946280212345000");
     verifyInsertValidationResponse(channel1.insertRow(row, null));
     row.put("ttzsmall", "2021-01-01 05:00:00 +0100");
     row.put("ttzbig", "2021-01-01 23:00:00.12345678 -0300");
     row.put("tsmall", "03:00:00.123");
     row.put("tbig", "11:00:00.12345678");
-    row.put("tntzsmall", "1809462800.123");
-    row.put("tntzbig", "2031-01-01 09:00:00.12345678");
+    row.put("tntzsmall", "1809462800123");
+    row.put("tntzbig", "2031-01-01 09:00:00.123456780");
     verifyInsertValidationResponse(channel1.insertRow(row, "1"));
 
     // Close the channel after insertion
@@ -502,7 +502,7 @@ public void testMultiColumnIngest() throws Exception {
     row.put("f", 3.14);
     row.put("tinyfloat", 1.1);
     row.put("var", "{\"e\":2.7}");
-    row.put("t", timestamp);
+    row.put("t", String.valueOf(timestamp));
     row.put("d", "1969-12-31 00:00:00");
     verifyInsertValidationResponse(channel1.insertRow(row, "1"));
 
@@ -1103,23 +1103,31 @@ private static Map<String, Object> getRandomRow(Random r) {
     row.put("var", nextJson(r));
     row.put("obj", nextJson(r));
     row.put("arr", Arrays.asList(r.nextInt(100), r.nextInt(100), r.nextInt(100)));
-    row.put("epochdays", Math.abs(r.nextInt()) % 18963); // DATE, 02.12.2021
+    row.put(
+        "epochdays",
+        String.valueOf(Math.abs(r.nextInt()) % (18963 * 24 * 60 * 60))); // DATE, 02.12.2021
     row.put(
         "timesec",
-        (r.nextInt(11) * 60 * 60 + r.nextInt(59) * 60 + r.nextInt(59)) * 10000
-            + r.nextInt(9999)); // TIME(4), 05:12:43.4536
+        String.valueOf(
+            (r.nextInt(11) * 60 * 60 + r.nextInt(59) * 60 + r.nextInt(59)) * 10000
+                + r.nextInt(9999))); // TIME(4), 05:12:43.4536
     row.put(
         "timenano",
-        (14 * 60 * 60 + 26 * 60 + 34) * 1000000000L + 437582643); // TIME(9), 14:26:34.437582643
+        String.valueOf(
+            (14 * 60 * 60 + 26 * 60 + 34) * 1000000000L
+                + 437582643)); // TIME(9), 14:26:34.437582643
     row.put(
-        "epochsec", Math.abs(r.nextLong()) % 1638459438); // TIMESTAMP_LTZ(0), 02.12.2021 15:37:18
+        "epochsec",
+        String.valueOf(
+            Math.abs(r.nextLong()) % 1638459438)); // TIMESTAMP_LTZ(0), 02.12.2021 15:37:18
     row.put(
         "epochnano",
-        new BigDecimal(
-            (Math.abs(r.nextInt()) % 1999999999)
-                + "."
+        String.format(
+            "%d%d",
+            Math.abs(r.nextInt()) % 1999999999,
+            100000000
                 + Math.abs(
-                    r.nextInt(999999999)))); // TIMESTAMP_LTZ(9), 18.05.2033 03:33:19.999999999
+                    r.nextInt(899999999)))); // TIMESTAMP_LTZ(9), 18.05.2033 03:33:19.999999999
     return row;
   }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
new file mode 100644
index 000000000..bad5eac16
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
@@ -0,0 +1,1721 @@
+package net.snowflake.ingest.streaming.internal.datatypes;
+
+import java.sql.SQLException;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.OffsetDateTime;
+import java.time.OffsetTime;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Supported date, time and timestamp formats:
+ * https://docs.snowflake.com/en/user-guide/date-time-input-output.html#date-formats
+ */
+public class DateTimeIT extends AbstractDataTypeTest {
+
+  @Test
+  public void testTimestampWithTimeZone() throws Exception {
+    useLosAngelesTimeZone();
+
+    // Test timestamp formats
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2/18/2008 02:36:48",
+        "2008-02-18 02:36:48.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28 20",
+        "2013-04-28 20:00:00.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28 20:57",
+        "2013-04-28 20:57:00.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28 20:57:01",
+        "2013-04-28 20:57:01.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28 20:57:01 +07:00",
+        "2013-04-28 20:57:01.000000000 +0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28 20:57:01 +0700",
+        "2013-04-28 20:57:01.000000000 +0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28 20:57:01-07",
+        "2013-04-28 20:57:01.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28 20:57:01-07:00",
+        "2013-04-28 20:57:01.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28 20:57:01.123456",
+        "2013-04-28 20:57:01.123456000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28 20:57:01.123456789 +07:00",
+        "2013-04-28 20:57:01.123456789 +0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28 20:57:01.123456789 +0700",
+        "2013-04-28 20:57:01.123456789 +0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28 20:57:01.123456789+07",
+        "2013-04-28 20:57:01.123456789 +0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28 20:57:01.123456789+07:00",
+        "2013-04-28 20:57:01.123456789 +0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28 20:57+07:00",
+        "2013-04-28 20:57:00.000000000 +0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28T20",
+        "2013-04-28 20:00:00.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28T20:57",
+        "2013-04-28 20:57:00.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28T20:57:01",
+        "2013-04-28 20:57:01.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28T20:57:01-07:00",
+        "2013-04-28 20:57:01.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28T20:57:01.123456",
+        "2013-04-28 20:57:01.123456000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28T20:57:01.123456789+07:00",
+        "2013-04-28 20:57:01.123456789 +0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28T20:57+07:00",
+        "2013-04-28 20:57:00.000000000 +0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "Mon Jul 08 18:09:51 +0000 2013",
+        "2013-07-08 18:09:51.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "Thu, 21 Dec 2000 04:01:07 PM",
+        "2000-12-21 16:01:07.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "Thu, 21 Dec 2000 04:01:07 PM +0200",
+        "2000-12-21 16:01:07.000000000 +0200",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "Thu, 21 Dec 2000 04:01:07.123456789 PM",
+        "2000-12-21 16:01:07.123456789 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "Thu, 21 Dec 2000 04:01:07.123456789 PM +0200",
+        "2000-12-21 16:01:07.123456789 +0200",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "Thu, 21 Dec 2000 16:01:07",
+        "2000-12-21 16:01:07.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "Thu, 21 Dec 2000 16:01:07 +0200",
+        "2000-12-21 16:01:07.000000000 +0200",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "Thu, 21 Dec 2000 16:01:07.123456789",
+        "2000-12-21 16:01:07.123456789 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "Thu, 21 Dec 2000 16:01:07.123456789 +0200",
+        "2000-12-21 16:01:07.123456789 +0200",
+        new StringProvider(),
+        new StringProvider());
+
+    // Test date formats
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2013-04-28",
+        "2013-04-28 00:00:00.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "17-DEC-1980",
+        "1980-12-17 00:00:00.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "12/17/1980",
+        "1980-12-17 00:00:00.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+
+    // Test leap years
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "2024-02-29T23:59:59.999999999Z",
+        "2024-02-29 23:59:59.999999999 Z",
+        new StringProvider(),
+        new StringProvider());
+    expectArrowNotSupported("TIMESTAMP_TZ", "2023-02-29T23:59:59.999999999Z");
+
+    // Test numeric strings
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "0",
+        "1970-01-01 00:00:00.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "86399",
+        "1970-01-01 23:59:59.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "86401",
+        "1970-01-02 00:00:01.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "-86401",
+        "1969-12-30 23:59:59.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "-86399",
+        "1969-12-31 00:00:01.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "1662731080",
+        "2022-09-09 13:44:40.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "1662731080123456789",
+        "2022-09-09 13:44:40.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+  }
+
+  @Test
+  public void testTimestampWithLocalTimeZone() throws Exception {
+    useLosAngelesTimeZone();
+
+    // Test timestamp formats
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2/18/2008 02:36:48",
+        "2008-02-18 02:36:48.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28 20",
+        "2013-04-28 20:00:00.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28 20:57",
+        "2013-04-28 20:57:00.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28 20:57:01",
+        "2013-04-28 20:57:01.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28 20:57:01 +07:00",
+        "2013-04-28 06:57:01.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28 20:57:01 +0700",
+        "2013-04-28 06:57:01.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28 20:57:01-07",
+        "2013-04-28 20:57:01.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28 20:57:01-07:00",
+        "2013-04-28 20:57:01.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28 20:57:01.123456",
+        "2013-04-28 20:57:01.123456000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28 20:57:01.123456789 +07:00",
+        "2013-04-28 06:57:01.123456789 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28 20:57:01.123456789 +0700",
+        "2013-04-28 06:57:01.123456789 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28 20:57:01.123456789+07",
+        "2013-04-28 06:57:01.123456789 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28 20:57:01.123456789+07:00",
+        "2013-04-28 06:57:01.123456789 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28 20:57+07:00",
+        "2013-04-28 06:57:00.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28T20",
+        "2013-04-28 20:00:00.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28T20:57",
+        "2013-04-28 20:57:00.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28T20:57:01",
+        "2013-04-28 20:57:01.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28T20:57:01-07:00",
+        "2013-04-28 20:57:01.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28T20:57:01.123456",
+        "2013-04-28 20:57:01.123456000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28T20:57:01.123456789+07:00",
+        "2013-04-28 06:57:01.123456789 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28T20:57+07:00",
+        "2013-04-28 06:57:00.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "Mon Jul 08 18:09:51 +0000 2013",
+        "2013-07-08 11:09:51.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "Thu, 21 Dec 2000 04:01:07 PM",
+        "2000-12-21 16:01:07.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "Thu, 21 Dec 2000 04:01:07 PM +0200",
+        "2000-12-21 06:01:07.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "Thu, 21 Dec 2000 04:01:07.123456789 PM",
+        "2000-12-21 16:01:07.123456789 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "Thu, 21 Dec 2000 04:01:07.123456789 PM +0200",
+        "2000-12-21 06:01:07.123456789 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "Thu, 21 Dec 2000 16:01:07",
+        "2000-12-21 16:01:07.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "Thu, 21 Dec 2000 16:01:07 +0200",
+        "2000-12-21 06:01:07.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "Thu, 21 Dec 2000 16:01:07.123456789",
+        "2000-12-21 16:01:07.123456789 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "Thu, 21 Dec 2000 16:01:07.123456789 +0200",
+        "2000-12-21 06:01:07.123456789 -0800",
+        new StringProvider(),
+        new StringProvider());
+
+    // Test date formats
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28",
+        "2013-04-28 00:00:00.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "17-DEC-1980",
+        "1980-12-17 00:00:00.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "12/17/1980",
+        "1980-12-17 00:00:00.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+
+    // Test boundary time zone
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28T00:00:00+07:00",
+        "2013-04-27 10:00:00.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2013-04-28T00:00:00-07:00",
+        "2013-04-28 00:00:00.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+
+    // Test leap years
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "2024-02-29T23:59:59.999999999Z",
+        "2024-02-29 15:59:59.999999999 -0800",
+        new StringProvider(),
+        new StringProvider());
+    expectArrowNotSupported("TIMESTAMP_LTZ", "2023-02-29T23:59:59.999999999Z");
+
+    // Test numeric strings
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "0",
+        "1969-12-31 16:00:00.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "86399",
+        "1970-01-01 15:59:59.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "86401",
+        "1970-01-01 16:00:01.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "-86401",
+        "1969-12-30 15:59:59.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "-86399",
+        "1969-12-30 16:00:01.000000000 -0800",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "1662731080",
+        "2022-09-09 06:44:40.000000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "1662731080123456789",
+        "2022-09-09 06:44:40.123456789 -0700",
+        new StringProvider(),
+        new StringProvider());
+  }
+
+  @Test
+  public void testTimestampWithoutTimeZone() throws Exception {
+    // Test timestamp formats
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2/18/2008 02:36:48",
+        "2008-02-18 02:36:48.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28 20",
+        "2013-04-28 20:00:00.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28 20:57",
+        "2013-04-28 20:57:00.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28 20:57:01",
+        "2013-04-28 20:57:01.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28 20:57:01 +07:00",
+        "2013-04-28 20:57:01.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28 20:57:01 +0700",
+        "2013-04-28 20:57:01.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28 20:57:01-07",
+        "2013-04-28 20:57:01.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28 20:57:01-07:00",
+        "2013-04-28 20:57:01.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28 20:57:01.123456",
+        "2013-04-28 20:57:01.123456000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28 20:57:01.123456789 +07:00",
+        "2013-04-28 20:57:01.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28 20:57:01.123456789 +0700",
+        "2013-04-28 20:57:01.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28 20:57:01.123456789+07",
+        "2013-04-28 20:57:01.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28 20:57:01.123456789+07:00",
+        "2013-04-28 20:57:01.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28 20:57+07:00",
+        "2013-04-28 20:57:00.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28T20",
+        "2013-04-28 20:00:00.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28T20:57",
+        "2013-04-28 20:57:00.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28T20:57:01",
+        "2013-04-28 20:57:01.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28T20:57:01-07:00",
+        "2013-04-28 20:57:01.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28T20:57:01.123456",
+        "2013-04-28 20:57:01.123456000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28T20:57:01.123456789+07:00",
+        "2013-04-28 20:57:01.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28T20:57+07:00",
+        "2013-04-28 20:57:00.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "Mon Jul 08 18:09:51 +0000 2013",
+        "2013-07-08 18:09:51.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "Thu, 21 Dec 2000 04:01:07 PM",
+        "2000-12-21 16:01:07.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "Thu, 21 Dec 2000 04:01:07 PM +0200",
+        "2000-12-21 16:01:07.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "Thu, 21 Dec 2000 04:01:07.123456789 PM",
+        "2000-12-21 16:01:07.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "Thu, 21 Dec 2000 04:01:07.123456789 PM +0200",
+        "2000-12-21 16:01:07.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "Thu, 21 Dec 2000 16:01:07",
+        "2000-12-21 16:01:07.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "Thu, 21 Dec 2000 16:01:07 +0200",
+        "2000-12-21 16:01:07.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "Thu, 21 Dec 2000 16:01:07.123456789",
+        "2000-12-21 16:01:07.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "Thu, 21 Dec 2000 16:01:07.123456789 +0200",
+        "2000-12-21 16:01:07.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+
+    // Test date formats
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28",
+        "2013-04-28 00:00:00.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "17-DEC-1980",
+        "1980-12-17 00:00:00.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "12/17/1980",
+        "1980-12-17 00:00:00.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+
+    // Test boundary time zone
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28T00:00:00+07:00",
+        "2013-04-28 00:00:00.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2013-04-28T00:00:00-07:00",
+        "2013-04-28 00:00:00.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+
+    // Test leap years
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "2024-02-29T23:59:59.999999999Z",
+        "2024-02-29 23:59:59.999999999 Z",
+        new StringProvider(),
+        new StringProvider());
+    expectArrowNotSupported("TIMESTAMP_NTZ", "2023-02-29T23:59:59.999999999Z");
+
+    // Test numeric strings
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "0",
+        "1970-01-01 00:00:00.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "86399",
+        "1970-01-01 23:59:59.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "86401",
+        "1970-01-02 00:00:01.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "-86401",
+        "1969-12-30 23:59:59.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "-86399",
+        "1969-12-31 00:00:01.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "1662731080",
+        "2022-09-09 13:44:40.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "1662731080123456789",
+        "2022-09-09 13:44:40.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+  }
+
+  @Test
+  public void testJavaTimeObjects() throws Exception {
+    // TIME (LocalTime and OffsetTime are supported)
+    testIngestion("TIME", LocalTime.of(23, 59, 59), "23:59:59.000000000 Z", new StringProvider());
+    testIngestion(
+        "TIME",
+        OffsetTime.of(23, 59, 59, 0, ZoneOffset.ofHoursMinutes(4, 0)),
+        "23:59:59.000000000 Z",
+        new StringProvider());
+    testIngestion(
+        "TIME",
+        OffsetTime.of(23, 59, 59, 0, ZoneOffset.ofHoursMinutes(-4, 0)),
+        "23:59:59.000000000 Z",
+        new StringProvider());
+    testIngestion(
+        "TIME",
+        OffsetTime.of(23, 59, 59, 123, ZoneOffset.ofHoursMinutes(-4, 0)),
+        "23:59:59.000000123 Z",
+        new StringProvider());
+    testIngestion(
+        "TIME",
+        OffsetTime.of(23, 59, 59, 123456789, ZoneOffset.ofHoursMinutes(-4, 0)),
+        "23:59:59.123456789 Z",
+        new StringProvider());
+
+    // DATE (LocalDate, LocalDateTime, OffsetDateTime, ZonedDateTime are supported)
+    testIngestion("DATE", LocalDate.parse("2007-12-03"), "2007-12-03", new StringProvider());
+    testIngestion(
+        "DATE", LocalDateTime.parse("2007-12-03T10:15:30"), "2007-12-03", new StringProvider());
+    testIngestion(
+        "DATE",
+        OffsetDateTime.parse("2007-12-03T10:15:30+01:00"),
+        "2007-12-03",
+        new StringProvider());
+    testIngestion(
+        "DATE",
+        ZonedDateTime.parse("2007-12-03T10:15:30+01:00[Europe/Paris]"),
+        "2007-12-03",
+        new StringProvider());
+
+    // TIMESTAMP_NTZ (LocalDate, LocalDateTime, OffsetDateTime, ZonedDateTime are supported)
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        LocalDate.parse("2007-12-03"),
+        "2007-12-03 00:00:00.000000000 Z",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        LocalDateTime.parse("2007-12-03T10:15:30"),
+        "2007-12-03 10:15:30.000000000 Z",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        OffsetDateTime.parse("2007-12-03T10:15:30+01:00"),
+        "2007-12-03 10:15:30.000000000 Z",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        ZonedDateTime.parse("2007-12-03T10:15:30+01:00[Europe/Paris]"),
+        "2007-12-03 10:15:30.000000000 Z",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        ZonedDateTime.parse("2007-12-03T10:15:30.123456789+01:00[Europe/Paris]"),
+        "2007-12-03 10:15:30.123456789 Z",
+        new StringProvider());
+
+    useLosAngelesTimeZone();
+    // TIMESTAMP_LTZ (LocalDate, LocalDateTime, OffsetDateTime, ZonedDateTime are supported)
+    testIngestion(
+        "TIMESTAMP_LTZ",
+        LocalDate.parse("2007-12-03"),
+        "2007-12-03 00:00:00.000000000 -0800",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_LTZ",
+        LocalDateTime.parse("2007-12-03T10:15:30"),
+        "2007-12-03 10:15:30.000000000 -0800",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_LTZ",
+        OffsetDateTime.parse("2007-12-03T10:15:30+01:00"),
+        "2007-12-03 01:15:30.000000000 -0800",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_LTZ",
+        ZonedDateTime.parse("2007-12-03T10:15:30+01:00[Europe/Paris]"),
+        "2007-12-03 01:15:30.000000000 -0800",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_LTZ",
+        ZonedDateTime.parse("2007-12-03T10:15:30.123456789+01:00[Europe/Paris]"),
+        "2007-12-03 01:15:30.123456789 -0800",
+        new StringProvider());
+
+    // TIMESTAMP_TZ (LocalDate, LocalDateTime, OffsetDateTime, ZonedDateTime are supported)
+    testIngestion(
+        "TIMESTAMP_TZ",
+        LocalDate.parse("2007-12-03"),
+        "2007-12-03 00:00:00.000000000 -0800",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_TZ",
+        LocalDateTime.parse("2007-12-03T10:15:30"),
+        "2007-12-03 10:15:30.000000000 -0800",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_TZ",
+        OffsetDateTime.parse("2007-12-03T10:15:30+01:00"),
+        "2007-12-03 10:15:30.000000000 +0100",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_TZ",
+        ZonedDateTime.parse("2007-12-03T10:15:30+01:00[Europe/Paris]"),
+        "2007-12-03 10:15:30.000000000 +0100",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_TZ",
+        ZonedDateTime.parse("2007-12-03T10:15:30.123456789+01:00[Europe/Paris]"),
+        "2007-12-03 10:15:30.123456789 +0100",
+        new StringProvider());
+  }
+
+  @Test
+  public void testTime() throws Exception {
+    // All (7) documented time formats are supported
+    // https://docs.snowflake.com/en/user-guide/date-time-input-output.html#time-formats
+
+    testJdbcTypeCompatibility(
+        "TIME",
+        "20:57:01.123456789+07:00",
+        "20:57:01.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME",
+        "20:57:01.123456789",
+        "20:57:01.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME", "20:57:01", "20:57:01.000000000 Z", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME", "20:57", "20:57:00.000000000 Z", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME",
+        "07:57:01.123456789 AM",
+        "07:57:01.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME", "04:01:07 AM", "04:01:07.000000000 Z", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME", "04:01 PM", "16:01:00.000000000 Z", new StringProvider(), new StringProvider());
+
+    // Test numeric strings
+    testJdbcTypeCompatibility(
+        "TIME", "0", "00:00:00.000000000 Z", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME", "86399", "23:59:59.000000000 Z", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME", "86401", "00:00:01.000000000 Z", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME", "-86401", "23:59:59.000000000 Z", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME", "-86399", "00:00:01.000000000 Z", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME", "1662731080", "13:44:40.000000000 Z", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME",
+        "1662731080123456789",
+        "13:44:40.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+  }
+
+  @Test
+  public void testLimitedScale() throws Exception {
+    // Of TIME
+    testJdbcTypeCompatibility(
+        "TIME(0)", "13:00:00.999", "13:00:00. Z", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME(0)", "13:00:00.999999999", "13:00:00. Z", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME(0)",
+        "13:00:00.999999999+07:30",
+        "13:00:00. Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME(0)",
+        "1662731080123456789",
+        "13:44:40. Z",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIME(4)", "13:00:00.999", "13:00:00.9990 Z", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME(4)",
+        "13:00:00.999999999",
+        "13:00:00.9999 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME(4)",
+        "13:00:00.999999999+07:30",
+        "13:00:00.9999 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME(4)",
+        "1662731080123456789",
+        "13:44:40.1234 Z",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIME(9)",
+        "13:00:00.999",
+        "13:00:00.999000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME(9)",
+        "13:00:00.999999999",
+        "13:00:00.999999999 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME(9)",
+        "13:00:00.999999999+07:30",
+        "13:00:00.999999999 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME(9)",
+        "1662731080123456789",
+        "13:44:40.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME(9)",
+        "1662731080123456",
+        "13:44:40.123456000 Z",
+        new StringProvider(),
+        new StringProvider());
+
+    // Of TIMESTAMP_NTZ
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(0)",
+        "2010-07-07 13:00:00.999",
+        "2010-07-07 13:00:00. Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(0)",
+        "2010-07-07 13:00:00.999999999",
+        "2010-07-07 13:00:00. Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(0)",
+        "2010-07-07 13:00:00.999999999+07:30",
+        "2010-07-07 13:00:00. Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(0)",
+        "1662731080123456789",
+        "2022-09-09 13:44:40. Z",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(4)",
+        "2010-07-07 13:00:00.999",
+        "2010-07-07 13:00:00.9990 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(4)",
+        "2010-07-07 13:00:00.999999999",
+        "2010-07-07 13:00:00.9999 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(4)",
+        "2010-07-07 13:00:00.999999999+07:30",
+        "2010-07-07 13:00:00.9999 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(4)",
+        "1662731080123456789",
+        "2022-09-09 13:44:40.1234 Z",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(7)",
+        "2010-07-07 13:00:00.999",
+        "2010-07-07 13:00:00.9990000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(7)",
+        "2010-07-07 13:00:00.999999999",
+        "2010-07-07 13:00:00.9999999 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(7)",
+        "2010-07-07 13:00:00.999999999+07:30",
+        "2010-07-07 13:00:00.9999999 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(7)",
+        "1662731080123456789",
+        "2022-09-09 13:44:40.1234567 Z",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(8)",
+        "2010-07-07 13:00:00.999",
+        "2010-07-07 13:00:00.99900000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(8)",
+        "2010-07-07 13:00:00.999999999",
+        "2010-07-07 13:00:00.99999999 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(8)",
+        "2010-07-07 13:00:00.999999999+07:30",
+        "2010-07-07 13:00:00.99999999 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(8)",
+        "1662731080123456789",
+        "2022-09-09 13:44:40.12345678 Z",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(9)",
+        "2010-07-07 13:00:00.999",
+        "2010-07-07 13:00:00.999000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(9)",
+        "2010-07-07 13:00:00.999999999",
+        "2010-07-07 13:00:00.999999999 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(9)",
+        "2010-07-07 13:00:00.999999999+07:30",
+        "2010-07-07 13:00:00.999999999 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ(9)",
+        "1662731080123456789",
+        "2022-09-09 13:44:40.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+
+    // Of TIMESTAMP_LTZ
+    useLosAngelesTimeZone();
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(0)",
+        "2010-07-07 13:00:00.999",
+        "2010-07-07 13:00:00. -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(0)",
+        "2010-07-07 13:00:00.999999999",
+        "2010-07-07 13:00:00. -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(0)",
+        "2010-07-07 13:00:00.999999999+07:30",
+        "2010-07-06 22:30:00. -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(0)",
+        "1662731080123456789",
+        "2022-09-09 06:44:40. -0700",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(4)",
+        "2010-07-07 13:00:00.999",
+        "2010-07-07 13:00:00.9990 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(4)",
+        "2010-07-07 13:00:00.999999999",
+        "2010-07-07 13:00:00.9999 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(4)",
+        "2010-07-07 13:00:00.999999999+07:30",
+        "2010-07-06 22:30:00.9999 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(4)",
+        "1662731080123456789",
+        "2022-09-09 06:44:40.1234 -0700",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(7)",
+        "2010-07-07 13:00:00.999",
+        "2010-07-07 13:00:00.9990000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(7)",
+        "2010-07-07 13:00:00.999999999",
+        "2010-07-07 13:00:00.9999999 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(7)",
+        "2010-07-07 13:00:00.999999999+07:30",
+        "2010-07-06 22:30:00.9999999 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(7)",
+        "1662731080123456789",
+        "2022-09-09 06:44:40.1234567 -0700",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(8)",
+        "2010-07-07 13:00:00.999",
+        "2010-07-07 13:00:00.99900000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(8)",
+        "2010-07-07 13:00:00.999999999",
+        "2010-07-07 13:00:00.99999999 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(8)",
+        "2010-07-07 13:00:00.999999999+07:30",
+        "2010-07-06 22:30:00.99999999 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(8)",
+        "1662731080123456789",
+        "2022-09-09 06:44:40.12345678 -0700",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(9)",
+        "2010-07-07 13:00:00.999",
+        "2010-07-07 13:00:00.999000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(9)",
+        "2010-07-07 13:00:00.999999999",
+        "2010-07-07 13:00:00.999999999 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(9)",
+        "2010-07-07 13:00:00.999999999+07:30",
+        "2010-07-06 22:30:00.999999999 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ(9)",
+        "1662731080123456789",
+        "2022-09-09 06:44:40.123456789 -0700",
+        new StringProvider(),
+        new StringProvider());
+
+    // Of TIMESTAMP_TZ
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(0)",
+        "2010-07-07 13:00:00.999",
+        "2010-07-07 13:00:00. -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(0)",
+        "2010-07-07 13:00:00.999999999",
+        "2010-07-07 13:00:00. -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(0)",
+        "2010-07-07 13:00:00.999999999+07:30",
+        "2010-07-07 13:00:00. +0730",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(0)",
+        "1662731080123456789",
+        "2022-09-09 13:44:40. Z",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(4)",
+        "2010-07-07 13:00:00.999",
+        "2010-07-07 13:00:00.9990 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(4)",
+        "2010-07-07 13:00:00.999999999",
+        "2010-07-07 13:00:00.9999 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(4)",
+        "2010-07-07 13:00:00.999999999+07:30",
+        "2010-07-07 13:00:00.9999 +0730",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(4)",
+        "1662731080123456789",
+        "2022-09-09 13:44:40.1234 Z",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(7)",
+        "2010-07-07 13:00:00.999",
+        "2010-07-07 13:00:00.9990000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(7)",
+        "2010-07-07 13:00:00.999999999",
+        "2010-07-07 13:00:00.9999999 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(7)",
+        "2010-07-07 13:00:00.999999999+07:30",
+        "2010-07-07 13:00:00.9999999 +0730",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(7)",
+        "1662731080123456789",
+        "2022-09-09 13:44:40.1234567 Z",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(8)",
+        "2010-07-07 13:00:00.999",
+        "2010-07-07 13:00:00.99900000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(8)",
+        "2010-07-07 13:00:00.999999999",
+        "2010-07-07 13:00:00.99999999 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(8)",
+        "2010-07-07 13:00:00.999999999+07:30",
+        "2010-07-07 13:00:00.99999999 +0730",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(8)",
+        "1662731080123456789",
+        "2022-09-09 13:44:40.12345678 Z",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(9)",
+        "2010-07-07 13:00:00.999",
+        "2010-07-07 13:00:00.999000000 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(9)",
+        "2010-07-07 13:00:00.999999999",
+        "2010-07-07 13:00:00.999999999 -0700",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(9)",
+        "2010-07-07 13:00:00.999999999+07:30",
+        "2010-07-07 13:00:00.999999999 +0730",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(9)",
+        "1662731080123456789",
+        "2022-09-09 13:44:40.123456789 Z",
+        new StringProvider(),
+        new StringProvider());
+  }
+
+  @Test
+  public void testDate() throws Exception {
+    // Test timestamp formats
+    testJdbcTypeCompatibility(
+        "DATE", "2/18/2008 02:36:48", "2008-02-18", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "2013-04-28 20", "2013-04-28", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "2013-04-28 20:57", "2013-04-28", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "2013-04-28 20:57:01", "2013-04-28", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "2013-04-28 20:57:01 +07:00",
+        "2013-04-28",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "2013-04-28 20:57:01 +0700",
+        "2013-04-28",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "2013-04-28 20:57:01-07", "2013-04-28", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "2013-04-28 20:57:01-07:00",
+        "2013-04-28",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "2013-04-28 20:57:01.123456",
+        "2013-04-28",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "2013-04-28 20:57:01.123456789 +07:00",
+        "2013-04-28",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "2013-04-28 20:57:01.123456789 +0700",
+        "2013-04-28",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "2013-04-28 20:57:01.123456789+07",
+        "2013-04-28",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "2013-04-28 20:57:01.123456789+07:00",
+        "2013-04-28",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "2013-04-28 20:57+07:00", "2013-04-28", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "2013-04-28T20", "2013-04-28", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "2013-04-28T20:57", "2013-04-28", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "2013-04-28T20:57:01", "2013-04-28", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "2013-04-28T20:57:01-07:00",
+        "2013-04-28",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "2013-04-28T20:57:01.123456",
+        "2013-04-28",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "2013-04-28T20:57:01.123456789+07:00",
+        "2013-04-28",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "2013-04-28T20:57+07:00", "2013-04-28", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "Mon Jul 08 18:09:51 +0000 2013",
+        "2013-07-08",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "Thu, 21 Dec 2000 04:01:07 PM",
+        "2000-12-21",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "Thu, 21 Dec 2000 04:01:07 PM +0200",
+        "2000-12-21",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "Thu, 21 Dec 2000 04:01:07.123456789 PM",
+        "2000-12-21",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "Thu, 21 Dec 2000 04:01:07.123456789 PM +0200",
+        "2000-12-21",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "Thu, 21 Dec 2000 16:01:07",
+        "2000-12-21",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "Thu, 21 Dec 2000 16:01:07 +0200",
+        "2000-12-21",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "Thu, 21 Dec 2000 16:01:07.123456789",
+        "2000-12-21",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "Thu, 21 Dec 2000 16:01:07.123456789 +0200",
+        "2000-12-21",
+        new StringProvider(),
+        new StringProvider());
+
+    // Test date formats
+    testJdbcTypeCompatibility(
+        "DATE", "2013-04-28", "2013-04-28", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "17-DEC-1980", "1980-12-17", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "12/17/1980", "1980-12-17", new StringProvider(), new StringProvider());
+
+    // Test leap years
+    testJdbcTypeCompatibility(
+        "DATE",
+        "2024-02-29T23:59:59.999999999Z",
+        "2024-02-29",
+        new StringProvider(),
+        new StringProvider());
+    expectArrowNotSupported("DATE", "2023-02-29T23:59:59.999999999Z");
+
+    testJdbcTypeCompatibility(
+        "DATE", "9999-12-31", "9999-12-31", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "9999-12-31T23:59:59.999999999Z",
+        "9999-12-31",
+        new StringProvider(),
+        new StringProvider());
+
+    // Test boundary date
+    testJdbcTypeCompatibility(
+        "DATE",
+        "2013-04-28T00:00:00+07:00",
+        "2013-04-28",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE",
+        "2013-04-28T00:00:00-07:00",
+        "2013-04-28",
+        new StringProvider(),
+        new StringProvider());
+
+    // Test numeric strings
+    testJdbcTypeCompatibility(
+        "DATE", "0", "1970-01-01", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "86399", "1970-01-01", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "86401", "1970-01-02", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "-86401", "1969-12-30", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "-86399", "1969-12-31", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "1662731080", "2022-09-09", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "1662731080123456789", "2022-09-09", new StringProvider(), new StringProvider());
+  }
+
+  @Test
+  @Ignore("SNOW-663646")
+  public void testOldValues() throws Exception {
+    testJdbcTypeCompatibility(
+        "DATE", "1582-01-01", "1582-10-01", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "1582-10-14", "1582-10-14", new StringProvider(), new StringProvider());
+  }
+
+  @Test
+  public void testFutureDates() throws Exception {
+    useLosAngelesTimeZone();
+
+    testJdbcTypeCompatibility(
+        "DATE", "99999-12-31", "99999-12-31", new StringProvider(), new StringProvider());
+
+    testJdbcTypeCompatibility( // SB16
+        "TIMESTAMP_NTZ",
+        "9999-12-31 23:59:59.999999999",
+        "9999-12-31 23:59:59.999999999 Z",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility( // SB8
+        "TIMESTAMP_NTZ(7)",
+        "9999-12-31 23:59:59.999999999",
+        "9999-12-31 23:59:59.9999999 Z",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility( // SB16
+        "TIMESTAMP_LTZ",
+        "9999-12-31 23:59:59.999999999",
+        "9999-12-31 23:59:59.999999999 -0800",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility( // SB8
+        "TIMESTAMP_LTZ(7)",
+        "9999-12-31 23:59:59.999999999",
+        "9999-12-31 23:59:59.9999999 -0800",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "9999-12-31 23:59:59.999999999",
+        "9999-12-31 23:59:59.999999999 -0800",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ(3)",
+        "9999-12-31 23:59:59.999999999",
+        "9999-12-31 23:59:59.999 -0800",
+        new StringProvider(),
+        new StringProvider());
+  }
+
+  /**
+   * To make assertions of timestamps without timezone to work, make sure JDBC connection session
+   * time zone is the same as the streaming ingest default timezone
+   */
+  private void useLosAngelesTimeZone() throws SQLException {
+    conn.createStatement().execute("alter session set timezone = 'America/Los_Angeles';");
+  }
+}

From aaf2a5ac2df8d345a53bf80af878c631bd3857b9 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Mon, 26 Sep 2022 10:15:45 +0200
Subject: [PATCH 064/356] SNOW-655614 Data type validation: NUMBER and REAL
 (#230)

* SNOW-655614 Data type validation: NUMBER and REAL

* Only limited of Java types are allowed, documented in Javadoc and exceptions
* Allowed range for NUMBER is determined by the logical type
    * Previously, the max allowed value for a column was determined by the max
      value, which fit into the physical type, which is failing in Snowflake.
* Half up rounding used when the scale of the value is greater than the column scale
---
 .../streaming/internal/ArrowRowBuffer.java    |  56 ++--
 .../internal/DataValidationUtil.java          | 227 +++-----------
 .../internal/DataValidationUtilTest.java      | 222 +++++---------
 .../streaming/internal/FlushServiceTest.java  |   1 +
 .../streaming/internal/RowBufferTest.java     |   5 +
 .../internal/datatypes/NumericTypesIT.java    | 282 ++++++++++++++++++
 6 files changed, 440 insertions(+), 353 deletions(-)
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 53a7ac10c..a8626bba7 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -6,6 +6,7 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.math.RoundingMode;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -14,7 +15,6 @@
 import java.util.Optional;
 import java.util.Set;
 import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
-import net.snowflake.client.jdbc.internal.snowflake.common.util.Power10;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
@@ -465,45 +465,47 @@ private float convertRowToArrow(
       } else {
         switch (logicalType) {
           case FIXED:
-            if (!field.getMetadata().get(COLUMN_SCALE).equals("0")
-                || physicalType == ColumnPhysicalType.SB16) {
-              int scale =
-                  DataValidationUtil.validateAndParseInteger(field.getMetadata().get(COLUMN_SCALE));
-              BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
-
-              // vector.setSafe requires the BigDecimal input scale explicitly match its scale
-              bigDecimalValue = bigDecimalValue.setScale(scale);
-              ((DecimalVector) vector).setSafe(curRowIndex, bigDecimalValue);
-              BigInteger intRep =
-                  bigDecimalValue
-                      .multiply(BigDecimal.valueOf(Power10.intTable[scale]))
-                      .toBigInteger();
-              stats.addIntValue(intRep);
+            int columnPrecision = Integer.parseInt(field.getMetadata().get(COLUMN_PRECISION));
+            int columnScale = getColumnScale(field.getMetadata());
+            BigDecimal inputAsBigDecimal = DataValidationUtil.validateAndParseBigDecimal(value);
+            // vector.setSafe requires the BigDecimal input scale explicitly match its scale
+            inputAsBigDecimal = inputAsBigDecimal.setScale(columnScale, RoundingMode.HALF_UP);
+
+            if (inputAsBigDecimal.abs().compareTo(BigDecimal.TEN.pow(columnPrecision - columnScale))
+                >= 0) {
+              throw new SFException(
+                  ErrorCode.INVALID_ROW,
+                  inputAsBigDecimal,
+                  String.format(
+                      "Number out of representable exclusive range of (-1e%s..1e%s)",
+                      columnPrecision - columnScale, columnPrecision - columnScale));
+            }
+
+            if (columnScale != 0 || physicalType == ColumnPhysicalType.SB16) {
+              ((DecimalVector) vector).setSafe(curRowIndex, inputAsBigDecimal);
+              stats.addIntValue(inputAsBigDecimal.unscaledValue());
               rowBufferSize += 16;
             } else {
               switch (physicalType) {
                 case SB1:
-                  byte byteValue = DataValidationUtil.validateAndParseByte(value);
-                  ((TinyIntVector) vector).setSafe(curRowIndex, byteValue);
-                  stats.addIntValue(BigInteger.valueOf(byteValue));
+                  ((TinyIntVector) vector).setSafe(curRowIndex, inputAsBigDecimal.byteValueExact());
+                  stats.addIntValue(inputAsBigDecimal.toBigInteger());
                   rowBufferSize += 1;
                   break;
                 case SB2:
-                  short shortValue = DataValidationUtil.validateAndParseShort(value);
-                  ((SmallIntVector) vector).setSafe(curRowIndex, shortValue);
-                  stats.addIntValue(BigInteger.valueOf(shortValue));
+                  ((SmallIntVector) vector)
+                      .setSafe(curRowIndex, inputAsBigDecimal.shortValueExact());
+                  stats.addIntValue(inputAsBigDecimal.toBigInteger());
                   rowBufferSize += 2;
                   break;
                 case SB4:
-                  int intVal = DataValidationUtil.validateAndParseInteger(value);
-                  ((IntVector) vector).setSafe(curRowIndex, intVal);
-                  stats.addIntValue(BigInteger.valueOf(intVal));
+                  ((IntVector) vector).setSafe(curRowIndex, inputAsBigDecimal.intValueExact());
+                  stats.addIntValue(inputAsBigDecimal.toBigInteger());
                   rowBufferSize += 4;
                   break;
                 case SB8:
-                  long longValue = DataValidationUtil.validateAndParseLong(value);
-                  ((BigIntVector) vector).setSafe(curRowIndex, longValue);
-                  stats.addIntValue(BigInteger.valueOf(longValue));
+                  ((BigIntVector) vector).setSafe(curRowIndex, inputAsBigDecimal.longValueExact());
+                  stats.addIntValue(inputAsBigDecimal.toBigInteger());
                   rowBufferSize += 8;
                   break;
                 default:
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 8f2baa563..26728cdd9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -282,34 +282,38 @@ static String validateAndParseString(Object input, Optional<Integer> maxLengthOp
 
   /**
    * Returns a BigDecimal representation of the input. Strings of the form "1.23E4" will be treated
-   * as being written in * scientific notation (e.g. 1.23 * 10^4)
-   *
-   * @param input
+   * as being written in * scientific notation (e.g. 1.23 * 10^4). Does not perform any size
+   * validation. Allowed Java types:
+   * <li>byte, short, int, long
+   * <li>float, double
+   * <li>BigInteger, BigDecimal
+   * <li>String
    */
   static BigDecimal validateAndParseBigDecimal(Object input) {
-    BigDecimal output;
-    try {
-      if (input instanceof BigDecimal) {
-        output = (BigDecimal) input;
-      } else if (input instanceof BigInteger) {
-        output = new BigDecimal((BigInteger) input);
-      } else if (input instanceof String) {
-        output = stringToBigDecimal((String) input);
-      } else {
-        output = stringToBigDecimal((input.toString()));
-      }
-      if (output.toBigInteger().compareTo(MAX_BIGINTEGER) >= 0
-          || output.toBigInteger().compareTo(MIN_BIGINTEGER) <= 0) {
-        throw new SFException(
-            ErrorCode.INVALID_ROW,
-            input.toString(),
-            String.format(
-                "Number out of representable range, Max=%s, Min=%s",
-                MIN_BIGINTEGER, MAX_BIGINTEGER));
+    if (input instanceof BigDecimal) {
+      return (BigDecimal) input;
+    } else if (input instanceof BigInteger) {
+      return new BigDecimal((BigInteger) input);
+    } else if (input instanceof Byte
+        || input instanceof Short
+        || input instanceof Integer
+        || input instanceof Long) {
+      return BigDecimal.valueOf(((Number) input).longValue());
+    } else if (input instanceof Float || input instanceof Double) {
+      return BigDecimal.valueOf(((Number) input).doubleValue());
+    } else if (input instanceof String) {
+      try {
+        return new BigDecimal((String) input);
+      } catch (NumberFormatException e) {
+        throw valueFormatNotAllowedException(input, "NUMBER");
       }
-      return output;
-    } catch (NumberFormatException e) {
-      throw new SFException(ErrorCode.INVALID_ROW, input.toString(), e.getMessage());
+    } else {
+      throw typeNotAllowedException(
+          input.getClass(),
+          "NUMBER",
+          new String[] {
+            "int", "long", "byte", "short", "float", "double", "BigDecimal", "BigInteger", "String"
+          });
     }
   }
 
@@ -323,57 +327,6 @@ static BigDecimal handleScientificNotationError(String input, NumberFormatExcept
         .multiply(BigDecimal.valueOf(10).pow(Integer.parseInt(splitInput[1])));
   }
 
-  static BigDecimal stringToBigDecimal(String input) {
-    try {
-      return new BigDecimal(input);
-    } catch (NumberFormatException e) {
-      return handleScientificNotationError(input, e);
-    }
-  }
-
-  /**
-   * Validates the input represents a valid NUMBER(38,0) and returns a BigInteger. Accepts Java
-   * number types and strings. Strings of the form "1.23E4" will be treated as being written in
-   * scientific notation (e.g. 1.23 * 10^4)
-   *
-   * @param input
-   */
-  static BigInteger validateAndParseBigInteger(Object input) {
-    try {
-      if (input instanceof Double) {
-        checkInteger((double) input);
-        return new BigDecimal(input.toString()).toBigInteger();
-      } else if (input instanceof Float) {
-        checkInteger((float) input);
-        return new BigDecimal(input.toString()).toBigInteger();
-      }
-      BigInteger output;
-      if (input instanceof BigInteger) {
-        output = (BigInteger) input;
-      } else if (input instanceof String) {
-        BigDecimal bigDecimal = stringToBigDecimal((String) input);
-        if (bigDecimal.stripTrailingZeros().scale() <= 0) {
-          return bigDecimal.toBigInteger();
-        } else {
-          throw new SFException(ErrorCode.INVALID_ROW, input.toString(), "Invalid integer");
-        }
-      } else {
-        output = new BigInteger(input.toString());
-      }
-      if (output.compareTo(MAX_BIGINTEGER) >= 0 || output.compareTo(MIN_BIGINTEGER) <= 0) {
-        throw new SFException(
-            ErrorCode.INVALID_ROW,
-            input.toString(),
-            String.format(
-                "Number out of representable range, Max=%s, Min=%s",
-                MIN_BIGINTEGER, MAX_BIGINTEGER));
-      }
-      return output;
-    } catch (NumberFormatException e) {
-      throw new SFException(ErrorCode.INVALID_ROW, input.toString(), e.getMessage());
-    }
-  }
-
   static void checkInteger(float input) {
     if (Math.floor(input) != input) {
       throw new SFException(ErrorCode.INVALID_ROW, input, "Value must be integer");
@@ -386,42 +339,6 @@ static void checkInteger(double input) {
     }
   }
 
-  /**
-   * Validates input is an integer @see {@link #validateAndParseInteger(Object)} and that it does
-   * not exceed the MIN/MAX Short value
-   *
-   * @param input
-   */
-  static short validateAndParseShort(Object input) {
-    Integer intValue = validateAndParseInteger(input);
-    if (intValue > Short.MAX_VALUE || intValue < Short.MIN_VALUE) {
-      throw new SFException(
-          ErrorCode.INVALID_ROW,
-          input.toString(),
-          String.format(
-              "Value exceeds min/max short.  min=%s, max%s", Short.MIN_VALUE, Short.MAX_VALUE));
-    }
-    return intValue.shortValue();
-  }
-
-  /**
-   * Validates input is an integer @see {@link #validateAndParseInteger(Object)} and that it does
-   * not exceed the MIN/MAX Byte value
-   *
-   * @param input
-   */
-  static byte validateAndParseByte(Object input) {
-    Integer intValue = validateAndParseInteger(input);
-    if (intValue > Byte.MAX_VALUE || intValue < Byte.MIN_VALUE) {
-      throw new SFException(
-          ErrorCode.INVALID_ROW,
-          input.toString(),
-          String.format(
-              "Value exceeds min/max byte.  min=%s, max%s", Byte.MIN_VALUE, Byte.MAX_VALUE));
-    }
-    return intValue.byteValue();
-  }
-
   /**
    * Validates the input can be represented as an integer with value between Integer.MIN_VALUE and
    * Integer.MAX_VALUE
@@ -481,62 +398,6 @@ static int validateAndParseInteger(Object input) {
     }
   }
 
-  /**
-   * Validates the input can be represented as an integer with value between Long.MIN_VALUE and
-   * Long.MAX_VALUE
-   *
-   * @param input
-   */
-  static long validateAndParseLong(Object input) {
-    try {
-      if (input instanceof Integer) {
-        return Long.valueOf((int) input);
-      } else if (input instanceof Long) {
-        return (long) input;
-      } else if (input instanceof Double) {
-        // Number must be integer
-        checkInteger((double) input);
-        BigInteger bigInt = validateAndParseBigInteger(input);
-        if (bigInt.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0
-            || bigInt.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) {
-          throw new SFException(
-              ErrorCode.INVALID_ROW, input.toString(), "Value greater than max long");
-        }
-        return ((Number) input).longValue();
-      } else if (input instanceof Float) {
-        // Number must be integer
-        checkInteger((float) input);
-        BigInteger bigInt = validateAndParseBigInteger(input);
-        if (bigInt.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0
-            || bigInt.compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) {
-          throw new SFException(
-              ErrorCode.INVALID_ROW, input.toString(), "Value greater than max long");
-        }
-        return ((Number) input).longValue();
-      } else if (input instanceof BigInteger) {
-        if (((BigInteger) input).compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0
-            || ((BigInteger) input).compareTo(BigInteger.valueOf(Long.MIN_VALUE)) < 0) {
-          throw new SFException(
-              ErrorCode.INVALID_ROW, input.toString(), "Value greater than max long");
-        }
-        return ((BigInteger) input).longValue();
-      } else {
-        try {
-          return Long.parseLong(input.toString());
-        } catch (NumberFormatException e) {
-          Double doubleValue = handleScientificNotationError(input.toString(), e).doubleValue();
-          if (Math.floor(doubleValue) == doubleValue) {
-            return doubleValue.longValue();
-          } else {
-            throw new SFException(ErrorCode.INVALID_ROW, input.toString(), "Value must be integer");
-          }
-        }
-      }
-    } catch (NumberFormatException err) {
-      throw new SFException(ErrorCode.INVALID_ROW, input.toString(), err.getMessage());
-    }
-  }
-
   /**
    * Returns the number of days between the epoch and the passed date. Allowed Java types:
    *
@@ -647,27 +508,29 @@ static String getStringValue(Object value) {
   }
 
   /**
-   * Converts input to double value.
+   * Converts input to double value. Allowed Java types:
+   *
+   * <ul>
+   *   <li>Number
+   *   <li>String
+   * </ul>
    *
    * @param input
    */
   static double validateAndParseReal(Object input) {
-    double doubleValue;
-    try {
-      if (input instanceof Double) {
-        doubleValue = (double) input;
-      } else if (input instanceof Float) {
-        doubleValue = Double.parseDouble(input.toString());
-      } else if (input instanceof BigDecimal) {
-        doubleValue = ((BigDecimal) input).doubleValue();
-      } else {
-        doubleValue = Double.parseDouble((String) input);
+    if (input instanceof Float) {
+      return Double.parseDouble(input.toString());
+    } else if (input instanceof Number) {
+      return ((Number) input).doubleValue();
+    } else if (input instanceof String) {
+      try {
+        return Double.parseDouble((String) input);
+      } catch (NumberFormatException err) {
+        throw valueFormatNotAllowedException(input, "REAL");
       }
-    } catch (NumberFormatException err) {
-      throw new SFException(ErrorCode.INVALID_ROW, input.toString());
     }
 
-    return doubleValue;
+    throw typeNotAllowedException(input.getClass(), "REAL", new String[] {"Number", "String"});
   }
 
   /**
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index a52008cf9..253f1ad21 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -1,8 +1,9 @@
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.streaming.internal.DataValidationUtil.MAX_BIGINTEGER;
+import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseBigDecimal;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseBoolean;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseDate;
+import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseReal;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseTime;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseTimestampNtzSb16;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseTimestampTz;
@@ -60,20 +61,6 @@ private void expectError(ErrorCode expectedErrorCode, Runnable action) {
     expectErrorCodeAndMessage(expectedErrorCode, null, action);
   }
 
-  @Test
-  public void testValidateAndParseShort() {
-    short e = 12;
-    Assert.assertEquals(e, DataValidationUtil.validateAndParseShort("12"));
-    Assert.assertEquals(e, DataValidationUtil.validateAndParseShort(e));
-    Assert.assertEquals(Short.MAX_VALUE, DataValidationUtil.validateAndParseShort(Short.MAX_VALUE));
-    Assert.assertEquals(Short.MIN_VALUE, DataValidationUtil.validateAndParseShort(Short.MIN_VALUE));
-
-    // Expect errors
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseShort, "howdy");
-    expectError(
-        ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseShort, Short.MAX_VALUE + 1);
-  }
-
   @Test
   public void testValidateAndParseTime() {
     assertEquals(5L, validateAndParseTime("00:00:05", 0).longValueExact());
@@ -277,74 +264,51 @@ public void testValidateAndPareTimestampTz() {
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz('c', 3));
   }
 
-  @Test
-  public void testValidateAndParseBigInteger() {
-    for (Object input : goodIntegersValue10) {
-      Assert.assertEquals(
-          new BigInteger("10"), DataValidationUtil.validateAndParseBigInteger(input));
-    }
-    Assert.assertEquals(
-        new BigInteger("-1000"), DataValidationUtil.validateAndParseBigInteger("-1e3"));
-
-    Assert.assertEquals(
-        BigInteger.valueOf(10).pow(37),
-        DataValidationUtil.validateAndParseBigInteger(BigInteger.valueOf(10).pow(37)));
-    Assert.assertEquals(
-        BigInteger.valueOf(-1).multiply(BigInteger.valueOf(10).pow(37)),
-        DataValidationUtil.validateAndParseBigInteger(
-            BigInteger.valueOf(-1).multiply(BigInteger.valueOf(10).pow(37))));
-
-    // Expect errors
-    // Too big
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseBigInteger,
-        BigInteger.valueOf(10).pow(38));
-    // Too small
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseBigInteger,
-        BigInteger.valueOf(-1).multiply(BigInteger.valueOf(10).pow(38)));
-    // Decimal
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigInteger, 1.1D);
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigInteger, 1.1F);
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigInteger, "1.1");
-  }
-
   @Test
   public void testValidateAndParseBigDecimal() {
-    Assert.assertEquals(new BigDecimal("1"), DataValidationUtil.validateAndParseBigDecimal("1"));
-    Assert.assertEquals(
-        new BigDecimal("1000").toBigInteger(),
-        DataValidationUtil.validateAndParseBigDecimal("1e3").toBigInteger());
-    Assert.assertEquals(
-        new BigDecimal("-1000").toBigInteger(),
-        DataValidationUtil.validateAndParseBigDecimal("-1e3").toBigInteger());
-    Assert.assertEquals(
-        new BigDecimal("1").toBigInteger(),
-        DataValidationUtil.validateAndParseBigDecimal("1e0").toBigInteger());
-    Assert.assertEquals(
-        new BigDecimal("-1").toBigInteger(),
-        DataValidationUtil.validateAndParseBigDecimal("-1e0").toBigInteger());
-    Assert.assertEquals(
-        new BigDecimal("123").toBigInteger(),
-        DataValidationUtil.validateAndParseBigDecimal("1.23e2").toBigInteger());
-    Assert.assertEquals(new BigDecimal("1"), DataValidationUtil.validateAndParseBigDecimal(1));
-    Assert.assertEquals(new BigDecimal("1.0"), DataValidationUtil.validateAndParseBigDecimal(1D));
-    Assert.assertEquals(new BigDecimal("1"), DataValidationUtil.validateAndParseBigDecimal(1L));
-    Assert.assertEquals(new BigDecimal("1.0"), DataValidationUtil.validateAndParseBigDecimal(1F));
-    Assert.assertEquals(
-        BigDecimal.valueOf(10).pow(37),
-        DataValidationUtil.validateAndParseBigDecimal(BigDecimal.valueOf(10).pow(37)));
-    Assert.assertEquals(
+    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal("1"));
+    assertEquals(
+        new BigDecimal("1000").toBigInteger(), validateAndParseBigDecimal("1e3").toBigInteger());
+    assertEquals(
+        new BigDecimal("-1000").toBigInteger(), validateAndParseBigDecimal("-1e3").toBigInteger());
+    assertEquals(
+        new BigDecimal("1").toBigInteger(), validateAndParseBigDecimal("1e0").toBigInteger());
+    assertEquals(
+        new BigDecimal("-1").toBigInteger(), validateAndParseBigDecimal("-1e0").toBigInteger());
+    assertEquals(
+        new BigDecimal("123").toBigInteger(), validateAndParseBigDecimal("1.23e2").toBigInteger());
+    assertEquals(
+        new BigDecimal("123.4").toBigInteger(),
+        validateAndParseBigDecimal("1.234e2").toBigInteger());
+    assertEquals(
+        new BigDecimal("0.1234").toBigInteger(),
+        validateAndParseBigDecimal("1.234e-1").toBigInteger());
+    assertEquals(
+        new BigDecimal("0.1234").toBigInteger(),
+        validateAndParseBigDecimal("1234e-5").toBigInteger());
+    assertEquals(
+        new BigDecimal("0.1234").toBigInteger(),
+        validateAndParseBigDecimal("1234E-5").toBigInteger());
+    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal(1));
+    assertEquals(new BigDecimal("1.0"), validateAndParseBigDecimal(1D));
+    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal(1L));
+    assertEquals(new BigDecimal("1.0"), validateAndParseBigDecimal(1F));
+    assertEquals(
+        BigDecimal.valueOf(10).pow(37), validateAndParseBigDecimal(BigDecimal.valueOf(10).pow(37)));
+    assertEquals(
         BigDecimal.valueOf(-1).multiply(BigDecimal.valueOf(10).pow(37)),
-        DataValidationUtil.validateAndParseBigDecimal(
+        validateAndParseBigDecimal(
             BigInteger.valueOf(-1).multiply(BigInteger.valueOf(10).pow(37))));
 
-    // Expect errors
+    // Test forbidden values
     expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigDecimal, "honk");
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigDecimal, "0x22");
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigDecimal, true);
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigDecimal, false);
     expectError(
-        ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigDecimal, MAX_BIGINTEGER);
+        ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigDecimal, new Object());
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigDecimal, 'a');
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigDecimal, new byte[4]);
   }
 
   @Test
@@ -523,60 +487,6 @@ public void testValidateAndParseInteger() {
         BigInteger.valueOf(Integer.MIN_VALUE).add(new BigInteger("-1")).toString());
   }
 
-  @Test
-  public void testValidateAndParseLong() {
-    for (Object input : goodIntegersValue10) {
-      Assert.assertEquals(10, DataValidationUtil.validateAndParseLong(input));
-    }
-
-    // Bad inputs
-    // Double
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseLong, 10.1D);
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseLong,
-        Double.valueOf(Long.MAX_VALUE) * 2);
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseLong,
-        Double.valueOf(Long.MIN_VALUE) * 2);
-
-    // Float
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseLong, 10.1F);
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseLong,
-        Float.valueOf(Long.MAX_VALUE) * 2);
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseLong,
-        Float.valueOf(Long.MIN_VALUE) * 2);
-
-    // BigInteger
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseLong,
-        BigInteger.valueOf(Long.MAX_VALUE).add(new BigInteger("1")));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseLong,
-        BigInteger.valueOf(Long.MIN_VALUE).add(new BigInteger("-1")));
-
-    // String
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseLong,
-        "Honk goes the noble goose");
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseLong,
-        BigInteger.valueOf(Long.MAX_VALUE).add(new BigInteger("1")).toString());
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseLong,
-        BigInteger.valueOf(Long.MIN_VALUE).add(new BigInteger("-1")).toString());
-  }
-
   @Test
   public void testValidateAndParseDate() {
     assertEquals(-923, validateAndParseDate("1967-06-23"));
@@ -658,23 +568,22 @@ public void testValidateAndParseBinary() {
   @Test
   public void testValidateAndParseReal() throws Exception {
     // From number types
-    Assert.assertEquals(1.23d, DataValidationUtil.validateAndParseReal(1.23f), 0);
-    Assert.assertEquals(1.23d, DataValidationUtil.validateAndParseReal(1.23), 0);
-    Assert.assertEquals(1.23d, DataValidationUtil.validateAndParseReal(1.23d), 0);
-    Assert.assertEquals(1.23d, DataValidationUtil.validateAndParseReal(new BigDecimal("1.23")), 0);
+    assertEquals(1.23d, validateAndParseReal(1.23f), 0);
+    assertEquals(1.23d, validateAndParseReal(1.23), 0);
+    assertEquals(1.23d, validateAndParseReal(1.23d), 0);
+    assertEquals(1.23d, validateAndParseReal(new BigDecimal("1.23")), 0);
 
     // From string
-    Assert.assertEquals(1.23d, DataValidationUtil.validateAndParseReal("1.23"), 0);
-    Assert.assertEquals(123d, DataValidationUtil.validateAndParseReal("1.23E2"), 0);
-    Assert.assertEquals(123d, DataValidationUtil.validateAndParseReal("1.23e2"), 0);
+    assertEquals(1.23d, validateAndParseReal("1.23"), 0);
+    assertEquals(123d, validateAndParseReal("1.23E2"), 0);
+    assertEquals(123d, validateAndParseReal("1.23e2"), 0);
 
-    // Error states
-    try {
-      DataValidationUtil.validateAndParseReal("honk");
-      Assert.fail("Expected invalid row error");
-    } catch (SFException err) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), err.getVendorCode());
-    }
+    // Test forbidden values
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseReal, "foo");
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseReal, 'c');
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseReal, new Object());
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseReal, false);
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseReal, true);
   }
 
   @Test
@@ -783,5 +692,30 @@ public void testExceptionMessages() {
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
             + " Snowflake column TIMESTAMP",
         () -> validateAndParseTimestampTz("abc", 3));
+
+    // NUMBER
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
+            + " be ingested into Snowflake column of type NUMBER. Allowed Java types: int, long,"
+            + " byte, short, float, double, BigDecimal, BigInteger, String",
+        () -> validateAndParseBigDecimal(new Object()));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
+            + " Snowflake column NUMBER",
+        () -> validateAndParseBigDecimal("abc"));
+
+    // REAL
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
+            + " be ingested into Snowflake column of type REAL. Allowed Java types: Number, String",
+        () -> validateAndParseReal(new Object()));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
+            + " Snowflake column REAL",
+        () -> validateAndParseReal("abc"));
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 2df8a1b19..80103e2df 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -336,6 +336,7 @@ private static ColumnMetadata createTestIntegerColumn() {
     colInt.setPhysicalType("SB4");
     colInt.setNullable(true);
     colInt.setLogicalType("FIXED");
+    colInt.setPrecision(2);
     colInt.setScale(0);
     return colInt;
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 75e053d10..7bd986e8a 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -52,6 +52,7 @@ static List<ColumnMetadata> createSchema() {
     colTinyIntCase.setPhysicalType("SB1");
     colTinyIntCase.setNullable(true);
     colTinyIntCase.setLogicalType("FIXED");
+    colTinyIntCase.setPrecision(2);
     colTinyIntCase.setScale(0);
 
     ColumnMetadata colTinyInt = new ColumnMetadata();
@@ -59,6 +60,7 @@ static List<ColumnMetadata> createSchema() {
     colTinyInt.setPhysicalType("SB1");
     colTinyInt.setNullable(true);
     colTinyInt.setLogicalType("FIXED");
+    colTinyInt.setPrecision(1);
     colTinyInt.setScale(0);
 
     ColumnMetadata colSmallInt = new ColumnMetadata();
@@ -66,6 +68,7 @@ static List<ColumnMetadata> createSchema() {
     colSmallInt.setPhysicalType("SB2");
     colSmallInt.setNullable(true);
     colSmallInt.setLogicalType("FIXED");
+    colSmallInt.setPrecision(2);
     colSmallInt.setScale(0);
 
     ColumnMetadata colInt = new ColumnMetadata();
@@ -73,6 +76,7 @@ static List<ColumnMetadata> createSchema() {
     colInt.setPhysicalType("SB4");
     colInt.setNullable(true);
     colInt.setLogicalType("FIXED");
+    colInt.setPrecision(2);
     colInt.setScale(0);
 
     ColumnMetadata colBigInt = new ColumnMetadata();
@@ -80,6 +84,7 @@ static List<ColumnMetadata> createSchema() {
     colBigInt.setPhysicalType("SB8");
     colBigInt.setNullable(true);
     colBigInt.setLogicalType("FIXED");
+    colBigInt.setPrecision(2);
     colBigInt.setScale(0);
 
     ColumnMetadata colDecimal = new ColumnMetadata();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
new file mode 100644
index 000000000..1b1c8d280
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
@@ -0,0 +1,282 @@
+package net.snowflake.ingest.streaming.internal.datatypes;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import org.junit.Test;
+
+public class NumericTypesIT extends AbstractDataTypeTest {
+  @Test
+  public void testIntegers() throws Exception {
+    // test bytes
+    testJdbcTypeCompatibility("INT", (short) 0, new ShortProvider());
+    testJdbcTypeCompatibility("INT", Short.MAX_VALUE, new ShortProvider());
+    testJdbcTypeCompatibility("INT", Short.MIN_VALUE, new ShortProvider());
+
+    // test shorts
+    testJdbcTypeCompatibility("INT", (byte) 0, new ByteProvider());
+    testJdbcTypeCompatibility("INT", Byte.MAX_VALUE, new ByteProvider());
+    testJdbcTypeCompatibility("INT", Byte.MIN_VALUE, new ByteProvider());
+
+    // test ints
+    testJdbcTypeCompatibility("INT", 0, new IntProvider());
+    testJdbcTypeCompatibility("INT", Integer.MAX_VALUE, new IntProvider());
+    testJdbcTypeCompatibility("INT", Integer.MIN_VALUE, new IntProvider());
+
+    // test longs
+    testJdbcTypeCompatibility("INT", 0L, new LongProvider());
+    testJdbcTypeCompatibility("INT", Long.MAX_VALUE, new LongProvider());
+    testJdbcTypeCompatibility("INT", Long.MIN_VALUE, new LongProvider());
+
+    // test floats
+    testJdbcTypeCompatibility("INT", 51.000f, 51, new FloatProvider(), new IntProvider());
+    testJdbcTypeCompatibility("INT", -51.000f, -51, new FloatProvider(), new IntProvider());
+    testJdbcTypeCompatibility("INT", 1.3f, 1, new FloatProvider(), new IntProvider());
+    testJdbcTypeCompatibility("INT", -1.3f, -1, new FloatProvider(), new IntProvider());
+
+    // test doubles
+    testJdbcTypeCompatibility("INT", 11.000, 11, new DoubleProvider(), new IntProvider());
+    testJdbcTypeCompatibility("INT", -11.000, -11, new DoubleProvider(), new IntProvider());
+    testJdbcTypeCompatibility("INT", 1.3, 1, new DoubleProvider(), new IntProvider());
+    testJdbcTypeCompatibility("INT", -1.3, -1, new DoubleProvider(), new IntProvider());
+
+    // test BigIntegers (JDBC does not support big integer, so we are reading back big decimals)
+    testIngestion(
+        "INT", MAX_ALLOWED_BIG_INTEGER, MAX_ALLOWED_BIG_DECIMAL, new BigDecimalProvider());
+    testIngestion(
+        "INT", MIN_ALLOWED_BIG_INTEGER, MIN_ALLOWED_BIG_DECIMAL, new BigDecimalProvider());
+    testIngestion("INT", BigInteger.ZERO, BigDecimal.ZERO, new BigDecimalProvider());
+    expectNumberOutOfRangeError("INT", MAX_ALLOWED_BIG_INTEGER.add(BigInteger.ONE), 38);
+    expectNumberOutOfRangeError("INT", MIN_ALLOWED_BIG_INTEGER.subtract(BigInteger.ONE), 38);
+
+    // test BigDecimals
+    testJdbcTypeCompatibility("INT", BigDecimal.ZERO, new BigDecimalProvider());
+    testJdbcTypeCompatibility("INT", MAX_ALLOWED_BIG_DECIMAL, new BigDecimalProvider());
+    testJdbcTypeCompatibility("INT", MIN_ALLOWED_BIG_DECIMAL, new BigDecimalProvider());
+    testJdbcTypeCompatibility("INT", new BigDecimal("51.000"), new BigDecimalProvider());
+    testJdbcTypeCompatibility("INT", new BigDecimal("-51.000"), new BigDecimalProvider());
+    expectNumberOutOfRangeError("INT", MAX_ALLOWED_BIG_DECIMAL.add(BigDecimal.ONE), 38);
+    expectNumberOutOfRangeError("INT", MIN_ALLOWED_BIG_DECIMAL.subtract(BigDecimal.ONE), 38);
+    testJdbcTypeCompatibility(
+        "INT",
+        new BigDecimal("9999999999999999999999999999999999.1234"),
+        new BigDecimal("9999999999999999999999999999999999"),
+        new BigDecimalProvider(),
+        new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "INT",
+        new BigDecimal("-9999999999999999999999999999999999.1234"),
+        new BigDecimal("-9999999999999999999999999999999999"),
+        new BigDecimalProvider(),
+        new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "INT",
+        new BigDecimal("0.4"),
+        new BigDecimal("0"),
+        new BigDecimalProvider(),
+        new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "INT",
+        new BigDecimal("0.6"),
+        new BigDecimal("1"),
+        new BigDecimalProvider(),
+        new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "INT",
+        new BigDecimal("-0.4"),
+        new BigDecimal("0"),
+        new BigDecimalProvider(),
+        new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "INT",
+        new BigDecimal("-0.6"),
+        new BigDecimal("-1"),
+        new BigDecimalProvider(),
+        new BigDecimalProvider());
+
+    // test Strings (Pass strings, read back as big decimal)
+    testJdbcTypeCompatibility(
+        "INT", "1", BigDecimal.ONE, new StringProvider(), new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "INT", "00001", BigDecimal.ONE, new StringProvider(), new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "INT", "8e0", new BigDecimal(8), new StringProvider(), new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "INT", "-10e4", new BigDecimal("-100000"), new StringProvider(), new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "INT", "11.5", new BigDecimal("12"), new StringProvider(), new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "INT", "12.4", new BigDecimal("12"), new StringProvider(), new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "INT", "1.2e1", new BigDecimal("12"), new StringProvider(), new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "INT", "1.25e1", new BigDecimal("13"), new StringProvider(), new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "INT", "4.000", new BigDecimal("4"), new StringProvider(), new BigDecimalProvider());
+    testIngestion(
+        "INT",
+        MAX_ALLOWED_BIG_INTEGER.toString(),
+        MAX_ALLOWED_BIG_DECIMAL,
+        new BigDecimalProvider());
+    testIngestion(
+        "INT",
+        MIN_ALLOWED_BIG_INTEGER.toString(),
+        MIN_ALLOWED_BIG_DECIMAL,
+        new BigDecimalProvider());
+  }
+
+  @Test
+  public void testNumbersWithLimitedPrecision() throws Exception {
+    testJdbcTypeCompatibility("NUMBER(1, 0)", (byte) 0, new ByteProvider());
+    testJdbcTypeCompatibility("NUMBER(1, 0)", (byte) 9, new ByteProvider());
+    testJdbcTypeCompatibility("NUMBER(1, 0)", (byte) -9, new ByteProvider());
+    expectNumberOutOfRangeError("NUMBER(1, 0)", 10, 1);
+    expectNumberOutOfRangeError("NUMBER(1, 0)", -10, 1);
+    expectNumberOutOfRangeError("NUMBER(1, 0)", -100, 1);
+    expectNumberOutOfRangeError("NUMBER(1, 0)", -1000, 1);
+  }
+
+  @Test
+  public void testNumbersWithScale() throws Exception {
+    testJdbcTypeCompatibility("NUMBER(1, 1)", 0, new IntProvider());
+    testJdbcTypeCompatibility("NUMBER(1, 1)", 0.00, new DoubleProvider());
+    testJdbcTypeCompatibility("NUMBER(1, 1)", 0.2, new DoubleProvider());
+    testJdbcTypeCompatibility("NUMBER(1, 1)", -0.2, new DoubleProvider());
+    testJdbcTypeCompatibility(
+        "NUMBER(1, 1)", 0.24, 0.2, new DoubleProvider(), new DoubleProvider());
+    testJdbcTypeCompatibility(
+        "NUMBER(1, 1)", -0.24, -0.2, new DoubleProvider(), new DoubleProvider());
+    testJdbcTypeCompatibility(
+        "NUMBER(1, 1)", 0.25, 0.3, new DoubleProvider(), new DoubleProvider());
+    testJdbcTypeCompatibility(
+        "NUMBER(1, 1)", -0.25, -0.3, new DoubleProvider(), new DoubleProvider());
+    expectNumberOutOfRangeError("NUMBER(1, 1)", 1, 0);
+    expectNumberOutOfRangeError("NUMBER(1, 1)", 10, 0);
+    expectNumberOutOfRangeError("NUMBER(1, 1)", 1.25, 0);
+
+    testJdbcTypeCompatibility("NUMBER(3, 1)", 0, new IntProvider());
+    testJdbcTypeCompatibility("NUMBER(3, 1)", 99, new IntProvider());
+    testJdbcTypeCompatibility("NUMBER(3, 1)", -99, new IntProvider());
+    testJdbcTypeCompatibility(
+        "NUMBER(3, 1)", 99.94, 99.9, new DoubleProvider(), new DoubleProvider());
+    testJdbcTypeCompatibility(
+        "NUMBER(3, 1)", -99.94, -99.9, new DoubleProvider(), new DoubleProvider());
+    expectNumberOutOfRangeError("NUMBER(3, 1)", 99.95, 2);
+    expectNumberOutOfRangeError("NUMBER(3, 1)", -99.95, 2);
+    expectNumberOutOfRangeError("NUMBER(3, 1)", 100, 2);
+    expectNumberOutOfRangeError("NUMBER(3, 1)", -100, 2);
+
+    testJdbcTypeCompatibility(
+        "NUMBER(38, 4)",
+        "9999999999999999999999999999999000",
+        new BigDecimal("9999999999999999999999999999999000"),
+        new StringProvider(),
+        new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "NUMBER(38, 4)",
+        "9999999999999999999999999999999999",
+        new BigDecimal("9999999999999999999999999999999999"),
+        new StringProvider(),
+        new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "NUMBER(38, 4)",
+        "9999999999999999999999999999999999.000",
+        new BigDecimal("9999999999999999999999999999999999"),
+        new StringProvider(),
+        new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "NUMBER(38, 4)",
+        "9999999999999999999999999999999999.9000",
+        new BigDecimal("9999999999999999999999999999999999.9"),
+        new StringProvider(),
+        new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "NUMBER(38, 4)",
+        "9999999999999999999999999999999999.9999",
+        new BigDecimal("9999999999999999999999999999999999.9999"),
+        new StringProvider(),
+        new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "NUMBER(38, 37)",
+        "0.0000000000000000000000000000000000000012",
+        new BigDecimal("0.0000000000000000000000000000000000000"),
+        new StringProvider(),
+        new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "NUMBER(38, 37)",
+        "9.9999999999999999999999999999999999999",
+        new BigDecimal("9.9999999999999999999999999999999999999"),
+        new StringProvider(),
+        new BigDecimalProvider());
+    expectNumberOutOfRangeError(
+        "NUMBER(38, 37)", new BigDecimal("9.99999999999999999999999999999999999999"), 1);
+
+    testJdbcTypeCompatibility(
+        "NUMBER(3, 1)",
+        "12.5",
+        new BigDecimal("12.5"),
+        new StringProvider(),
+        new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "NUMBER(3, 1)",
+        "1.25e1",
+        new BigDecimal("12.5"),
+        new StringProvider(),
+        new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "NUMBER(2, 0)",
+        "12.5",
+        new BigDecimal("13"),
+        new StringProvider(),
+        new BigDecimalProvider());
+    testJdbcTypeCompatibility(
+        "NUMBER(2, 0)",
+        "1.25e1",
+        new BigDecimal("13"),
+        new StringProvider(),
+        new BigDecimalProvider());
+  }
+
+  @Test
+  public void testFloatingPointTypes() throws Exception {
+    // Test double
+    testJdbcTypeCompatibility("REAL", 1.35, 1.35, new DoubleProvider(), new DoubleProvider());
+    testJdbcTypeCompatibility(
+        "REAL", Double.NaN, Double.NaN, new DoubleProvider(), new DoubleProvider());
+    testJdbcTypeCompatibility(
+        "REAL",
+        Double.NEGATIVE_INFINITY,
+        Double.NEGATIVE_INFINITY,
+        new DoubleProvider(),
+        new DoubleProvider());
+    testJdbcTypeCompatibility(
+        "REAL",
+        Double.POSITIVE_INFINITY,
+        Double.POSITIVE_INFINITY,
+        new DoubleProvider(),
+        new DoubleProvider());
+
+    // Test float
+    testJdbcTypeCompatibility("REAL", 1.35f, 1.35, new FloatProvider(), new DoubleProvider());
+    testJdbcTypeCompatibility(
+        "REAL",
+        Float.NEGATIVE_INFINITY,
+        Double.NEGATIVE_INFINITY,
+        new FloatProvider(),
+        new DoubleProvider());
+    testJdbcTypeCompatibility(
+        "REAL",
+        Float.POSITIVE_INFINITY,
+        Double.POSITIVE_INFINITY,
+        new FloatProvider(),
+        new DoubleProvider());
+    testJdbcTypeCompatibility(
+        "REAL", Float.NaN, Double.NaN, new FloatProvider(), new DoubleProvider());
+
+    // Test others
+    testJdbcTypeCompatibility("REAL", 1, 1., new IntProvider(), new DoubleProvider());
+    testJdbcTypeCompatibility("REAL", 1L, 1., new LongProvider(), new DoubleProvider());
+    testJdbcTypeCompatibility("REAL", "1.35", 1.35, new StringProvider(), new DoubleProvider());
+    testIngestion("REAL", new BigDecimal("1.35"), new BigDecimal("1.35"), new BigDecimalProvider());
+    testIngestion("REAL", BigInteger.ONE, BigDecimal.ONE, new BigDecimalProvider());
+  }
+}

From 5193d9a79c4cd8001f66e2e2afd5042f76b96703 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Wed, 28 Sep 2022 10:49:47 +0200
Subject: [PATCH 065/356] SNOW-655614 Data type validation: String and Binary
 (#231)

SNOW-655614 Data type validation: String and Binary

* Limited types allowed for BINARY: byte[] and hex-encoded string
* Limited types allowed for STRING: String, numbers, boolean and char
* Unified exception handling
    * Added explanations why are values being rejected
* Additional test cases added
---
 .../streaming/internal/ArrowRowBuffer.java    |   8 +-
 .../internal/DataValidationUtil.java          | 211 +++++++---------
 .../java/net/snowflake/ingest/TestUtils.java  |   2 +-
 .../internal/DataValidationUtilTest.java      | 233 +++++++++---------
 .../internal/datatypes/BinaryIT.java          |  31 +++
 .../internal/datatypes/StringsIT.java         |  90 +++++++
 6 files changed, 336 insertions(+), 239 deletions(-)
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index a8626bba7..38b4583a9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -520,9 +520,7 @@ private float convertRowToArrow(
               String maxLengthString = field.getMetadata().get(COLUMN_CHAR_LENGTH);
               String str =
                   DataValidationUtil.validateAndParseString(
-                      value,
-                      Optional.ofNullable(maxLengthString)
-                          .map(s -> DataValidationUtil.validateAndParseInteger(maxLengthString)));
+                      value, Optional.ofNullable(maxLengthString).map(Integer::parseInt));
               Text text = new Text(str);
               ((VarCharVector) vector).setSafe(curRowIndex, text);
               stats.addStrValue(str);
@@ -726,9 +724,7 @@ private float convertRowToArrow(
             String maxLengthString = field.getMetadata().get(COLUMN_BYTE_LENGTH);
             byte[] bytes =
                 DataValidationUtil.validateAndParseBinary(
-                    value,
-                    Optional.ofNullable(maxLengthString)
-                        .map(s -> DataValidationUtil.validateAndParseInteger(maxLengthString)));
+                    value, Optional.ofNullable(maxLengthString).map(Integer::parseInt));
             ((VarBinaryVector) vector).setSafe(curRowIndex, bytes);
             stats.addStrValue(new String(bytes, StandardCharsets.UTF_8));
             rowBufferSize += bytes.length;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 26728cdd9..59cd21ba6 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -34,12 +34,8 @@
 
 /** Utility class for parsing and validating inputs based on Snowflake types */
 class DataValidationUtil {
-  static final int MAX_STRING_LENGTH = 16 * 1024 * 1024;
-  static final int MAX_BINARY_LENGTH = 8 * 1024 * 1024;
-
-  static final BigInteger MAX_BIGINTEGER = BigInteger.valueOf(10).pow(38);
-  static final BigInteger MIN_BIGINTEGER =
-      BigInteger.valueOf(-1).multiply(BigInteger.valueOf(10).pow(38));
+  public static final int BYTES_8_MB = 8 * 1024 * 1024;
+  public static final int BYTES_16_MB = 2 * BYTES_8_MB;
 
   private static final TimeZone DEFAULT_TIMEZONE =
       TimeZone.getTimeZone("America/Los_Angeles"); // default value of TIMEZONE system parameter
@@ -69,12 +65,11 @@ static String validateAndParseVariant(Object input) {
           e, ErrorCode.INVALID_ROW, input, "Input column can't be convert to String.");
     }
 
-    if (output.length() > MAX_STRING_LENGTH) {
+    if (output.length() > BYTES_16_MB) {
       throw new SFException(
           ErrorCode.INVALID_ROW,
           input.toString(),
-          String.format(
-              "Variant too long: length=%d maxLength=%d", output.length(), MAX_STRING_LENGTH));
+          String.format("Variant too long: length=%d maxLength=%d", output.length(), BYTES_16_MB));
     }
     return output;
   }
@@ -102,12 +97,11 @@ static String validateAndParseArray(Object input) {
     }
 
     // Throw an exception if the size is too large
-    if (output.length() > MAX_STRING_LENGTH) {
+    if (output.length() > BYTES_16_MB) {
       throw new SFException(
           ErrorCode.INVALID_ROW,
           input.toString(),
-          String.format(
-              "Array too large. length=%d maxLength=%d", output.length(), MAX_STRING_LENGTH));
+          String.format("Array too large. length=%d maxLength=%d", output.length(), BYTES_16_MB));
     }
     return output;
   }
@@ -127,12 +121,11 @@ static String validateAndParseObject(Object input) {
           e, ErrorCode.INVALID_ROW, input.toString(), "Input column can't be convert to Json");
     }
 
-    if (output.length() > MAX_STRING_LENGTH) {
+    if (output.length() > BYTES_16_MB) {
       throw new SFException(
           ErrorCode.INVALID_ROW,
           input.toString(),
-          String.format(
-              "Object too large. length=%d maxLength=%d", output.length(), MAX_STRING_LENGTH));
+          String.format("Object too large. length=%d maxLength=%d", output.length(), BYTES_16_MB));
     }
     return output;
   }
@@ -192,7 +185,12 @@ else if (input instanceof OffsetDateTime)
               .add(BigInteger.valueOf(fraction));
       return new TimestampWrapper(epoch, fraction, timeInScale);
     } else {
-      throw valueFormatNotAllowedException(input.toString(), "TIMESTAMP");
+      throw valueFormatNotAllowedException(
+          input.toString(),
+          "TIMESTAMP",
+          "Not a valid timestamp, see"
+              + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
+              + " for the list of supported formats");
     }
   }
 
@@ -256,12 +254,25 @@ else if (input instanceof OffsetDateTime)
           BigInteger.valueOf(epoch * Power10.intTable[scale] + fraction),
           timestamp);
     } else {
-      throw valueFormatNotAllowedException(input.toString(), "TIMESTAMP");
+      throw valueFormatNotAllowedException(
+          input.toString(),
+          "TIMESTAMP",
+          "Not a valid timestamp, see"
+              + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
+              + " for the list of supported formats");
     }
   }
 
   /**
-   * Validates the input is less than the supposed maximum allowed string
+   * Converts input to string, validates that length is less than max allowed string size
+   * https://docs.snowflake.com/en/sql-reference/data-types-text.html#varchar. Allowed data types:
+   *
+   * <ul>
+   *   <li>String
+   *   <li>Number
+   *   <li>boolean
+   *   <li>char
+   * </ul>
    *
    * @param input Object to validate and parse to String
    * @param maxLengthOptional Maximum allowed length of the output String, if empty then uses
@@ -269,17 +280,27 @@ else if (input instanceof OffsetDateTime)
    *     (https://docs.snowflake.com/en/sql-reference/data-types-text.html#varchar)
    */
   static String validateAndParseString(Object input, Optional<Integer> maxLengthOptional) {
-    int maxLength = maxLengthOptional.orElse(MAX_STRING_LENGTH);
-    String output = getStringValue(input);
+    String output;
+    if (input instanceof String) {
+      output = (String) input;
+    } else if (input instanceof Number) {
+      output = new BigDecimal(input.toString()).stripTrailingZeros().toPlainString();
+    } else if (input instanceof Boolean || input instanceof Character) {
+      output = input.toString();
+    } else {
+      throw typeNotAllowedException(
+          input.getClass(), "STRING", new String[] {"String", "Number", "boolean", "char"});
+    }
+    int maxLength = maxLengthOptional.orElse(BYTES_16_MB);
+
     if (output.length() > maxLength) {
-      throw new SFException(
-          ErrorCode.INVALID_ROW,
-          input.toString(),
+      throw valueFormatNotAllowedException(
+          input,
+          "STRING",
           String.format("String too long: length=%d maxLength=%d", output.length(), maxLength));
     }
     return output;
   }
-
   /**
    * Returns a BigDecimal representation of the input. Strings of the form "1.23E4" will be treated
    * as being written in * scientific notation (e.g. 1.23 * 10^4). Does not perform any size
@@ -305,7 +326,7 @@ static BigDecimal validateAndParseBigDecimal(Object input) {
       try {
         return new BigDecimal((String) input);
       } catch (NumberFormatException e) {
-        throw valueFormatNotAllowedException(input, "NUMBER");
+        throw valueFormatNotAllowedException(input, "NUMBER", "Not a valid number");
       }
     } else {
       throw typeNotAllowedException(
@@ -317,87 +338,6 @@ static BigDecimal validateAndParseBigDecimal(Object input) {
     }
   }
 
-  static BigDecimal handleScientificNotationError(String input, NumberFormatException e) {
-    // Support scientific notation on string input
-    String[] splitInput = input.toLowerCase().split("e");
-    if (splitInput.length != 2) {
-      throw e;
-    }
-    return (new BigDecimal(splitInput[0]))
-        .multiply(BigDecimal.valueOf(10).pow(Integer.parseInt(splitInput[1])));
-  }
-
-  static void checkInteger(float input) {
-    if (Math.floor(input) != input) {
-      throw new SFException(ErrorCode.INVALID_ROW, input, "Value must be integer");
-    }
-  }
-
-  static void checkInteger(double input) {
-    if (Math.floor(input) != input) {
-      throw new SFException(ErrorCode.INVALID_ROW, input, "Value must be integer");
-    }
-  }
-
-  /**
-   * Validates the input can be represented as an integer with value between Integer.MIN_VALUE and
-   * Integer.MAX_VALUE
-   *
-   * @param input
-   */
-  static int validateAndParseInteger(Object input) {
-    try {
-      if (input instanceof Integer) {
-        return (int) input;
-      } else if (input instanceof Long) {
-        if ((long) input > Integer.MAX_VALUE || (long) input < Integer.MIN_VALUE) {
-          throw new SFException(
-              ErrorCode.INVALID_ROW, input.toString(), "Value greater than max integer");
-        }
-        return ((Long) input).intValue();
-      } else if (input instanceof Double) {
-        // Number must be integer
-        checkInteger((double) input);
-        if (((Number) input).longValue() > Integer.MAX_VALUE
-            || ((Number) input).longValue() < Integer.MIN_VALUE) {
-          throw new SFException(
-              ErrorCode.INVALID_ROW, input.toString(), "Value greater than max integer");
-        }
-        return ((Number) input).intValue();
-      } else if (input instanceof Float) {
-        // Number must be integer
-        checkInteger((float) input);
-
-        if (((Number) input).longValue() > Integer.MAX_VALUE
-            || ((Number) input).longValue() < Integer.MIN_VALUE) {
-          throw new SFException(
-              ErrorCode.INVALID_ROW, input.toString(), "Value greater than max integer");
-        }
-        return ((Number) input).intValue();
-      } else if (input instanceof BigInteger) {
-        if (((BigInteger) input).compareTo(BigInteger.valueOf(Integer.MAX_VALUE)) > 0
-            || ((BigInteger) input).compareTo(BigInteger.valueOf(Integer.MIN_VALUE)) < 0) {
-          throw new SFException(
-              ErrorCode.INVALID_ROW, input.toString(), "Value greater than max integer");
-        }
-        return ((BigInteger) input).intValue();
-      } else {
-        try {
-          return Integer.parseInt(input.toString());
-        } catch (NumberFormatException e) {
-          Double doubleValue = handleScientificNotationError(input.toString(), e).doubleValue();
-          if (Math.floor(doubleValue) == doubleValue) {
-            return doubleValue.intValue();
-          } else {
-            throw new SFException(ErrorCode.INVALID_ROW, input.toString(), "Value must be integer");
-          }
-        }
-      }
-    } catch (NumberFormatException err) {
-      throw new SFException(ErrorCode.INVALID_ROW, input.toString(), err.getMessage());
-    }
-  }
-
   /**
    * Returns the number of days between the epoch and the passed date. Allowed Java types:
    *
@@ -430,28 +370,49 @@ static int validateAndParseDate(Object input) {
 
     SFTimestamp timestamp =
         createDateTimeFormatter().parse(inputString, GMT, 0, DATE | TIMESTAMP, true, null);
-    if (timestamp == null) throw valueFormatNotAllowedException(input, "DATE");
+    if (timestamp == null)
+      throw valueFormatNotAllowedException(
+          input,
+          "DATE",
+          "Not a valid date, see"
+              + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#date-formats"
+              + " for the list of supported formats");
 
     return (int) TimeUnit.MILLISECONDS.toDays(SFDate.fromTimestamp(timestamp).getTime());
   }
 
+  /**
+   * Validates input for data type BINARY. Allowed Java types:
+   *
+   * <ul>
+   *   <li>byte[]
+   *   <li>String (hex-encoded)
+   * </ul>
+   *
+   * @param input Array to validate
+   * @param maxLengthOptional Max array length, defaults to 8MB, which is the max allowed length for
+   *     BINARY column
+   * @return Validated array
+   */
   static byte[] validateAndParseBinary(Object input, Optional<Integer> maxLengthOptional) {
     byte[] output;
     if (input instanceof byte[]) {
       output = (byte[]) input;
-    } else {
+    } else if (input instanceof String) {
       try {
-        output = DatatypeConverter.parseHexBinary(input.toString());
+        output = DatatypeConverter.parseHexBinary((String) input);
       } catch (IllegalArgumentException e) {
-        throw new SFException(ErrorCode.INVALID_ROW, input, e.getMessage());
+        throw valueFormatNotAllowedException(input, "BINARY", "Not a valid hex string");
       }
+    } else {
+      throw typeNotAllowedException(input.getClass(), "BINARY", new String[] {"byte[]", "String"});
     }
 
-    int maxLength = maxLengthOptional.orElse(MAX_BINARY_LENGTH);
+    int maxLength = maxLengthOptional.orElse(BYTES_8_MB);
     if (output.length > maxLength) {
-      throw new SFException(
-          ErrorCode.INVALID_ROW,
-          input.toString(),
+      throw valueFormatNotAllowedException(
+          String.format("byte[%d]", output.length),
+          "BINARY",
           String.format("Binary too long: length=%d maxLength=%d", output.length, maxLength));
     }
     return output;
@@ -484,7 +445,12 @@ static BigInteger validateAndParseTime(Object input, int scale) {
         createDateTimeFormatter()
             .parse(stringInput, GMT, 0, SnowflakeDateTimeFormat.TIME, true, null);
     if (timestamp == null) {
-      throw valueFormatNotAllowedException(input, "TIME");
+      throw valueFormatNotAllowedException(
+          input,
+          "TIME",
+          "Not a valid time, see"
+              + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#time-formats"
+              + " for the list of supported formats");
     } else {
       return timestamp
           .getNanosSinceEpoch()
@@ -526,7 +492,7 @@ static double validateAndParseReal(Object input) {
       try {
         return Double.parseDouble((String) input);
       } catch (NumberFormatException err) {
-        throw valueFormatNotAllowedException(input, "REAL");
+        throw valueFormatNotAllowedException(input, "REAL", "Not a valid decimal number");
       }
     }
 
@@ -566,7 +532,12 @@ static int validateAndParseBoolean(Object input) {
   private static boolean convertStringToBoolean(String value) {
     String lowerCasedValue = value.toLowerCase();
     if (!allowedBooleanStringsLowerCased.contains(lowerCasedValue)) {
-      throw valueFormatNotAllowedException(value, "BOOLEAN");
+      throw valueFormatNotAllowedException(
+          value,
+          "BOOLEAN",
+          "Not a valid boolean, see"
+              + " https://docs.snowflake.com/en/sql-reference/data-types-logical.html#conversion-to-boolean"
+              + " for the list of supported formats");
     }
     return "1".equals(lowerCasedValue)
         || "yes".equals(lowerCasedValue)
@@ -601,11 +572,13 @@ private static SFException typeNotAllowedException(
    * @param value Invalid value causing the exception
    * @param snowflakeType Snowflake column type
    */
-  private static SFException valueFormatNotAllowedException(Object value, String snowflakeType) {
+  private static SFException valueFormatNotAllowedException(
+      Object value, String snowflakeType, String reason) {
     return new SFException(
         ErrorCode.INVALID_ROW,
         sanitizeValueForExceptionMessage(value),
-        String.format("Value cannot be ingested into Snowflake column %s", snowflakeType));
+        String.format(
+            "Value cannot be ingested into Snowflake column %s: %s", snowflakeType, reason));
   }
 
   /**
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 0e2d3f145..5e5e594cd 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -224,7 +224,7 @@ public static Connection getConnection() throws Exception {
    * @throws Exception
    */
   public static Connection getConnection(boolean isStreamingConnection) throws Exception {
-    if (!isStreamingConnection && snowpipeConn != null && !streamingConn.isClosed()) {
+    if (!isStreamingConnection && snowpipeConn != null && !snowpipeConn.isClosed()) {
       return snowpipeConn;
     }
     if (isStreamingConnection && streamingConn != null && !streamingConn.isClosed()) {
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 253f1ad21..27ecc992a 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -1,12 +1,17 @@
 package net.snowflake.ingest.streaming.internal;
 
+import static net.snowflake.ingest.streaming.internal.DataValidationUtil.BYTES_16_MB;
+import static net.snowflake.ingest.streaming.internal.DataValidationUtil.BYTES_8_MB;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseBigDecimal;
+import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseBinary;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseBoolean;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseDate;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseReal;
+import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseString;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseTime;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseTimestampNtzSb16;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseTimestampTz;
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 
 import com.fasterxml.jackson.databind.JsonNode;
@@ -36,9 +41,6 @@
 public class DataValidationUtilTest {
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
-  private static final Object[] goodIntegersValue10 =
-      new Object[] {10D, 10F, 10L, new BigInteger("10"), 10, "10", "1e1", "1.0e1"};
-
   private void expectError(ErrorCode expectedErrorCode, Function func, Object args) {
     expectError(expectedErrorCode, () -> func.apply(args));
   }
@@ -313,29 +315,38 @@ public void testValidateAndParseBigDecimal() {
 
   @Test
   public void testValidateAndParseString() {
-    Assert.assertEquals(
-        "honk", DataValidationUtil.validateAndParseString("honk", Optional.empty()));
+    assertEquals("honk", validateAndParseString("honk", Optional.empty()));
 
     // Check max String length
     StringBuilder longBuilder = new StringBuilder();
-    for (int i = 0; i < DataValidationUtil.MAX_STRING_LENGTH + 1; i++) {
-      longBuilder.append("a");
+    for (int i = 0; i < BYTES_16_MB; i++) {
+      longBuilder.append("č"); // max string length is measured in chars, not bytes
     }
-    String tooLong = longBuilder.toString();
+    String maxString = longBuilder.toString();
+    Assert.assertEquals(maxString, validateAndParseString(maxString, Optional.empty()));
 
-    try {
-      DataValidationUtil.validateAndParseString(tooLong, Optional.empty());
-      Assert.fail("Expected error for String too long");
-    } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
-    }
+    // max length - 1 should also succeed
+    longBuilder.setLength(BYTES_16_MB - 1);
+    String maxStringMinusOne = longBuilder.toString();
+    Assert.assertEquals(
+        maxStringMinusOne, validateAndParseString(maxStringMinusOne, Optional.empty()));
 
-    try {
-      DataValidationUtil.validateAndParseString("123", Optional.of(2));
-      Assert.fail("Expected error for String too long");
-    } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
-    }
+    // max length + 1 should fail
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseString(longBuilder.append("aa").toString(), Optional.empty()));
+
+    // Test max length validation
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString("12345", Optional.of(4)));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString(false, Optional.of(4)));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString(12345, Optional.of(4)));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString(1.2345, Optional.of(4)));
+
+    // Test unsupported values
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseString(new Object(), Optional.empty()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString(new byte[] {}, Optional.of(4)));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString(new char[] {}, Optional.of(4)));
   }
 
   @Test
@@ -423,70 +434,6 @@ public void testValidateAndParseObject() throws Exception {
     }
   }
 
-  @Test
-  public void testValidateAndParseInteger() {
-    for (Object input : goodIntegersValue10) {
-      Assert.assertEquals(10, DataValidationUtil.validateAndParseInteger(input));
-    }
-
-    // Bad inputs
-    // Double
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseInteger, 10.1D);
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseInteger,
-        Double.valueOf(Integer.MAX_VALUE) + 1);
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseInteger,
-        Double.valueOf(Integer.MIN_VALUE) - 1);
-
-    // Float
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseInteger, 10.1F);
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseInteger,
-        Float.valueOf(Integer.MAX_VALUE) * 2);
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseInteger,
-        Float.valueOf(Integer.MIN_VALUE) * 2);
-
-    // Long
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseInteger,
-        Long.valueOf(Integer.MAX_VALUE) + 1);
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseInteger,
-        Long.valueOf(Integer.MIN_VALUE) - 1);
-
-    // BigInteger
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseInteger,
-        BigInteger.valueOf(Integer.MAX_VALUE).add(new BigInteger("1")));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseInteger,
-        BigInteger.valueOf(Integer.MIN_VALUE).add(new BigInteger("-1")));
-
-    // String
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseInteger,
-        "Honk goes the noble goose");
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseInteger,
-        BigInteger.valueOf(Integer.MAX_VALUE).add(new BigInteger("1")).toString());
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseInteger,
-        BigInteger.valueOf(Integer.MIN_VALUE).add(new BigInteger("-1")).toString());
-  }
-
   @Test
   public void testValidateAndParseDate() {
     assertEquals(-923, validateAndParseDate("1967-06-23"));
@@ -537,32 +484,49 @@ public void testGetStringValue() throws Exception {
 
   @Test
   public void testValidateAndParseBinary() {
-    Assert.assertArrayEquals(
+    byte[] maxAllowedArray = new byte[BYTES_8_MB];
+    byte[] maxAllowedArrayMinusOne = new byte[BYTES_8_MB - 1];
+
+    assertArrayEquals(
         "honk".getBytes(StandardCharsets.UTF_8),
-        DataValidationUtil.validateAndParseBinary(
-            "honk".getBytes(StandardCharsets.UTF_8), Optional.empty()));
+        validateAndParseBinary("honk".getBytes(StandardCharsets.UTF_8), Optional.empty()));
 
-    Assert.assertArrayEquals(
-        DatatypeConverter.parseHexBinary("12"),
-        DataValidationUtil.validateAndParseBinary("12", Optional.empty()));
+    assertArrayEquals(
+        new byte[] {-1, 0, 1}, validateAndParseBinary(new byte[] {-1, 0, 1}, Optional.empty()));
+    assertArrayEquals(
+        DatatypeConverter.parseHexBinary(
+            "1234567890abcdef"), // pragma: allowlist secret NOT A SECRET
+        validateAndParseBinary(
+            "1234567890abcdef", Optional.empty())); // pragma: allowlist secret NOT A SECRET
 
-    Assert.assertArrayEquals(
-        DatatypeConverter.parseHexBinary("12"),
-        DataValidationUtil.validateAndParseBinary(12, Optional.empty()));
+    assertArrayEquals(maxAllowedArray, validateAndParseBinary(maxAllowedArray, Optional.empty()));
+    assertArrayEquals(
+        maxAllowedArrayMinusOne, validateAndParseBinary(maxAllowedArrayMinusOne, Optional.empty()));
 
-    try {
-      DataValidationUtil.validateAndParseBinary("1212", Optional.of(1));
-      Assert.fail("Expected error for Binary too long");
-    } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
-    }
+    // Too large arrays should be rejected
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary(new byte[1], Optional.of(0)));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseBinary(new byte[BYTES_8_MB + 1], Optional.empty()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary(new byte[8], Optional.of(7)));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("aabb", Optional.of(1)));
 
-    try {
-      DataValidationUtil.validateAndParseBinary(123, Optional.empty());
-      Assert.fail("Expected error for invalid Binary format");
-    } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
-    }
+    // unsupported data types should fail
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("000", Optional.empty()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("abcg", Optional.empty()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("c", Optional.empty()));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () ->
+            validateAndParseBinary(Arrays.asList((byte) 1, (byte) 2, (byte) 3), Optional.empty()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary(1, Optional.empty()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary(12, Optional.empty()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary(1.5, Optional.empty()));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseBinary(BigInteger.ONE, Optional.empty()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary(false, Optional.empty()));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseBinary(new Object(), Optional.empty()));
   }
 
   @Test
@@ -625,7 +589,9 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column BOOLEAN",
+            + " Snowflake column BOOLEAN: Not a valid boolean, see"
+            + " https://docs.snowflake.com/en/sql-reference/data-types-logical.html#conversion-to-boolean"
+            + " for the list of supported formats",
         () -> validateAndParseBoolean("abc"));
 
     // TIME
@@ -638,7 +604,9 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column TIME",
+            + " Snowflake column TIME: Not a valid time, see"
+            + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#time-formats"
+            + " for the list of supported formats",
         () -> validateAndParseTime("abc", 10));
 
     // DATE
@@ -651,7 +619,9 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column DATE",
+            + " Snowflake column DATE: Not a valid date, see"
+            + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#date-formats"
+            + " for the list of supported formats",
         () -> validateAndParseDate("abc"));
 
     // TIMESTAMP_NTZ
@@ -664,7 +634,9 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column TIMESTAMP",
+            + " Snowflake column TIMESTAMP: Not a valid timestamp, see"
+            + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
+            + " for the list of supported formats",
         () -> validateAndParseTimestampNtzSb16("abc", 3, true));
 
     // TIMESTAMP_LTZ
@@ -677,7 +649,9 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column TIMESTAMP",
+            + " Snowflake column TIMESTAMP: Not a valid timestamp, see"
+            + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
+            + " for the list of supported formats",
         () -> validateAndParseTimestampNtzSb16("abc", 3, false));
 
     // TIMESTAMP_TZ
@@ -690,7 +664,9 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column TIMESTAMP",
+            + " Snowflake column TIMESTAMP: Not a valid timestamp, see"
+            + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
+            + " for the list of supported formats",
         () -> validateAndParseTimestampTz("abc", 3));
 
     // NUMBER
@@ -703,7 +679,7 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column NUMBER",
+            + " Snowflake column NUMBER: Not a valid number",
         () -> validateAndParseBigDecimal("abc"));
 
     // REAL
@@ -715,7 +691,38 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column REAL",
+            + " Snowflake column REAL: Not a valid decimal number",
         () -> validateAndParseReal("abc"));
+
+    // STRING
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
+            + " be ingested into Snowflake column of type STRING. Allowed Java types: String,"
+            + " Number, boolean, char",
+        () -> validateAndParseString(new Object(), Optional.empty()));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
+            + " Snowflake column STRING: String too long: length=3 maxLength=2",
+        () -> validateAndParseString("abc", Optional.of(2)));
+
+    // BINARY
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
+            + " be ingested into Snowflake column of type BINARY. Allowed Java types: byte[],"
+            + " String",
+        () -> validateAndParseBinary(new Object(), Optional.empty()));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: byte[2]. Value cannot be ingested into"
+            + " Snowflake column BINARY: Binary too long: length=2 maxLength=1",
+        () -> validateAndParseBinary(new byte[] {1, 2}, Optional.of(1)));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: ghi. Value cannot be ingested into"
+            + " Snowflake column BINARY: Not a valid hex string",
+        () -> validateAndParseBinary("ghi", Optional.empty()));
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
new file mode 100644
index 000000000..f0b0d8199
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
@@ -0,0 +1,31 @@
+package net.snowflake.ingest.streaming.internal.datatypes;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class BinaryIT extends AbstractDataTypeTest {
+
+  @Test
+  public void testBinarySimple() throws Exception {
+    testJdbcTypeCompatibility("BINARY", new byte[0], new ByteArrayProvider());
+    testJdbcTypeCompatibility("BINARY", new byte[3], new ByteArrayProvider());
+
+    testJdbcTypeCompatibility("BINARY", new byte[8 * 1024 * 1024], new ByteArrayProvider());
+
+    testJdbcTypeCompatibility("BINARY", new byte[] {1, 2, 3, 4}, new ByteArrayProvider());
+    testJdbcTypeCompatibility(
+        "BINARY", "212D", new byte[] {33, 45}, new StringProvider(), new ByteArrayProvider());
+    testJdbcTypeCompatibility(
+        "BINARY",
+        "0F00030A057F",
+        new byte[] {15, 0, 3, 10, 5, 127},
+        new StringProvider(),
+        new ByteArrayProvider());
+  }
+
+  @Test
+  @Ignore("SNOW-663704")
+  public void testBinary() throws Exception {
+    testJdbcTypeCompatibility("BINARY", new byte[] {-1}, new ByteArrayProvider());
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
new file mode 100644
index 000000000..4bcf39519
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
@@ -0,0 +1,90 @@
+package net.snowflake.ingest.streaming.internal.datatypes;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class StringsIT extends AbstractDataTypeTest {
+  @Test
+  public void testStrings() throws Exception {
+    testJdbcTypeCompatibility("VARCHAR", "", new StringProvider());
+    testJdbcTypeCompatibility("VARCHAR", "foo", new StringProvider());
+
+    // Test strings with limited size
+    testJdbcTypeCompatibility("VARCHAR(2)", "", new StringProvider());
+    testJdbcTypeCompatibility("VARCHAR(2)", "ab", new StringProvider());
+    expectArrowNotSupported("VARCHAR(2)", "abc");
+
+    // test booleans
+    testJdbcTypeCompatibility("CHAR(5)", true, "true", new BooleanProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "CHAR(5)", false, "false", new BooleanProvider(), new StringProvider());
+    expectArrowNotSupported("CHAR(4)", false);
+
+    // test numbers
+    testJdbcTypeCompatibility(
+        "CHAR(4)", (byte) 123, "123", new ByteProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "CHAR(4)", (short) 1111, "1111", new ShortProvider(), new StringProvider());
+    testJdbcTypeCompatibility("CHAR(4)", 1111, "1111", new IntProvider(), new StringProvider());
+    testJdbcTypeCompatibility("CHAR(4)", 1111L, "1111", new LongProvider(), new StringProvider());
+    testIngestion("CHAR(4)", BigInteger.valueOf(1111), "1111", new StringProvider());
+    testJdbcTypeCompatibility("CHAR(3)", 1.5f, "1.5", new FloatProvider(), new StringProvider());
+    testJdbcTypeCompatibility("CHAR(3)", 1.500f, "1.5", new FloatProvider(), new StringProvider());
+    testJdbcTypeCompatibility("CHAR(3)", 1.5d, "1.5", new DoubleProvider(), new StringProvider());
+    testJdbcTypeCompatibility("CHAR(3)", 1.500d, "1.5", new DoubleProvider(), new StringProvider());
+
+    // BigDecimal
+    testJdbcTypeCompatibility(
+        "CHAR(4)",
+        BigDecimal.valueOf(1111),
+        "1111",
+        new BigDecimalProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "CHAR(1)", new BigDecimal("4.0000"), "4", new BigDecimalProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "VARCHAR",
+        new BigDecimal("4e10"),
+        "40000000000",
+        new BigDecimalProvider(),
+        new StringProvider());
+
+    // char
+    testIngestion("CHAR(4)", 'c', "c", new StringProvider());
+  }
+
+  @Test
+  public void testNonAsciiStrings() throws Exception {
+    testIngestion(
+        "VARCHAR", "ž, š, č, ř, c, j, ď, ť, ň", "ž, š, č, ř, c, j, ď, ť, ň", new StringProvider());
+  }
+
+  @Test
+  public void testMaxAllowedString() throws Exception {
+    StringBuilder maxAllowedStringBuilder = buildString('a', 16 * 1024 * 1024);
+    String maxString = maxAllowedStringBuilder.toString();
+    testIngestion("VARCHAR", maxString, maxString, new StringProvider());
+    expectArrowNotSupported("VARCHAR", maxAllowedStringBuilder.append('a').toString());
+  }
+
+  @Test
+  @Ignore("SNOW-663621")
+  public void testMaxAllowedMultibyteString() throws Exception {
+    String times16 = "čččččččččččččččč";
+    String times17 = "ččččččččččččččččč";
+    testIngestion("VARCHAR", times16, times16, new StringProvider()); // works fine
+    testIngestion("VARCHAR", times17, times17, new StringProvider()); // fails
+    //    expectArrowNotSupported("VARCHAR",
+    // maxAllowedMultibyteStringBuilder.append('a').toString());
+  }
+
+  private StringBuilder buildString(char character, int count) {
+    StringBuilder maxStringBuilder = new StringBuilder(count);
+    for (int i = 0; i < count; i++) {
+      maxStringBuilder.append(character);
+    }
+    return maxStringBuilder;
+  }
+}

From 1270fdd4d07d8d9fd685e44640a00c1a4d64d672 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 28 Sep 2022 15:01:47 -0700
Subject: [PATCH 066/356] SNOW-616916: throttle the insertRows API when the
 flush task queue size reaches a certain threshold (#215)

We haven seen a few cases where the SDK OOM because of slow uploading speed causing everything to be hold in memory, the fix here is to throttle the insertRows API when the flush task queue reaches a certain threshold. Once the upload finishes and the task completed, the queue would be able to accept more rows.

Note that this PR also updates some of the constant values to tolerant more low memory cases
---
 .../streaming/internal/AbstractRowBuffer.java |   4 +-
 .../streaming/internal/FlushService.java      |  34 +++++-
 .../streaming/internal/RegisterService.java   |  32 ++++--
 ...owflakeStreamingIngestChannelInternal.java |  63 +++++-----
 .../net/snowflake/ingest/utils/Constants.java |   5 +-
 .../ingest/utils/ParameterProvider.java       | 108 +++++++++++++-----
 .../net/snowflake/ingest/utils/Utils.java     |  41 +++++++
 .../internal/ParameterProviderTest.java       |  29 +++--
 .../SnowflakeStreamingIngestClientTest.java   |   8 +-
 .../streaming/internal/StreamingIngestIT.java |  12 +-
 10 files changed, 237 insertions(+), 99 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 2adacd14d..064967e43 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -357,8 +357,8 @@ public ChannelData<T> flush() {
       logger.logDebug(
           "Arrow buffer flush released lock on channel={}, rowCount={}, bufferSize={}",
           this.owningChannel.getFullyQualifiedName(),
-          rowCount,
-          bufferSize);
+          oldRowCount,
+          oldBufferSize);
 
       if (oldData.isPresent()) {
         ChannelData<T> data = new ChannelData<>();
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index ba7c61b4c..8a4d83df0 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -5,7 +5,6 @@
 package net.snowflake.ingest.streaming.internal;
 
 import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.CPU_IO_TIME_RATIO;
 import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
 import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
 import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
@@ -31,6 +30,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.zip.CRC32;
@@ -291,13 +291,14 @@ private void createWorkers() {
     this.registerWorker = Executors.newSingleThreadExecutor(registerThreadFactory);
 
     // Create threads for building and uploading blobs
-    // Size: number of available processors * (1 + IO time/CPU time), currently the
-    // CPU_IO_TIME_RATIO is 1 under the assumption that build and upload will take similar time
+    // Size: number of available processors * (1 + IO time/CPU time)
     ThreadFactory buildUploadThreadFactory =
         new ThreadFactoryBuilder().setNameFormat("ingest-build-upload-thread-%d").build();
     int buildUploadThreadCount =
         Math.min(
-            Runtime.getRuntime().availableProcessors() * (1 + CPU_IO_TIME_RATIO), MAX_THREAD_COUNT);
+            Runtime.getRuntime().availableProcessors()
+                * (1 + this.owningClient.getParameterProvider().getIOTimeCpuRatio()),
+            MAX_THREAD_COUNT);
     this.buildUploadWorkers =
         Executors.newFixedThreadPool(buildUploadThreadCount, buildUploadThreadFactory);
 
@@ -355,8 +356,6 @@ void distributeFlushTasks() {
                 CompletableFuture.supplyAsync(
                     () -> {
                       try {
-                        logger.logDebug(
-                            "buildUploadWorkers stats={}", this.buildUploadWorkers.toString());
                         return buildAndUpload(filePath, blobData);
                       } catch (IOException e) {
                         String errorMessage =
@@ -383,6 +382,11 @@ void distributeFlushTasks() {
                       }
                     },
                     this.buildUploadWorkers)));
+        logger.logInfo(
+            "buildAndUpload task added for client={}, blob={}, buildUploadWorkers stats={}",
+            this.owningClient.getName(),
+            filePath,
+            this.buildUploadWorkers.toString());
       }
     }
 
@@ -624,4 +628,22 @@ <CD> void invalidateAllChannelsInBlob(List<List<ChannelData<CD>>> blobData) {
   String getClientPrefix() {
     return this.targetStage.getClientPrefix();
   }
+
+  /**
+   * Throttle if the number of queued buildAndUpload tasks is bigger than the total number of
+   * available processors
+   */
+  boolean throttleDueToQueuedFlushTasks() {
+    ThreadPoolExecutor buildAndUpload = (ThreadPoolExecutor) this.buildUploadWorkers;
+    boolean throttleOnQueuedTasks =
+        buildAndUpload.getQueue().size() > Runtime.getRuntime().availableProcessors();
+    if (throttleOnQueuedTasks) {
+      logger.logWarn(
+          "Throttled due too many queue flush tasks (probably because of slow uploading speed),"
+              + " client={}, buildUploadWorkers stats={}",
+          this.owningClient.getName(),
+          this.buildUploadWorkers.toString());
+    }
+    return throttleOnQueuedTasks;
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index 844852be5..4887441c0 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -4,7 +4,6 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_MAX_RETRY_COUNT;
 import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
 
 import com.codahale.metrics.Timer;
@@ -16,6 +15,7 @@
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.Pair;
 import net.snowflake.ingest.utils.Utils;
@@ -49,7 +49,7 @@ class RegisterService<T> {
    *
    * @param client
    */
-  RegisterService(SnowflakeStreamingIngestClientInternal client, boolean isTestMode) {
+  RegisterService(SnowflakeStreamingIngestClientInternal<T> client, boolean isTestMode) {
     this.owningClient = client;
     this.blobsList = new ArrayList<>();
     this.blobsListLock = new ReentrantLock();
@@ -109,9 +109,14 @@ List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyT
         // next blob if time out has been reached.
         int idx = 0;
         int retry = 0;
+        logger.logDebug(
+            "Start loop outer for uploading blobs={}",
+            oldList.stream().map(blob -> blob.getKey().getFilePath()).collect(Collectors.toList()));
         while (idx < oldList.size()) {
           List<BlobMetadata> blobs = new ArrayList<>();
           long startTime = System.currentTimeMillis();
+          logger.logDebug(
+              "Start loop inner for uploading blobs, size={}, idx={}", oldList.size(), idx);
           while (idx < oldList.size()
               && System.currentTimeMillis() - startTime
                   <= TimeUnit.SECONDS.toMillis(BLOB_UPLOAD_TIMEOUT_IN_SEC * 2)) {
@@ -119,12 +124,16 @@ List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyT
                 oldList.get(idx);
             try {
               logger.logDebug(
-                  "Start waiting on uploading blob={}", futureBlob.getKey().getFilePath());
+                  "Start waiting on uploading blob={}, idx={}",
+                  futureBlob.getKey().getFilePath(),
+                  idx);
               // Wait for uploading to finish
               BlobMetadata blob =
                   futureBlob.getValue().get(BLOB_UPLOAD_TIMEOUT_IN_SEC, TimeUnit.SECONDS);
               logger.logDebug(
-                  "Finish waiting on uploading blob={}", futureBlob.getKey().getFilePath());
+                  "Finish waiting on uploading blob={}, idx={}",
+                  futureBlob.getKey().getFilePath(),
+                  idx);
               if (blob != null) {
                 blobs.add(blob);
               }
@@ -137,9 +146,13 @@ List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyT
               // blobs are generated before these channels got invalidated.
 
               // Retry logic for timeout exception only
-              if (e instanceof TimeoutException && retry < BLOB_UPLOAD_MAX_RETRY_COUNT) {
+              if (e instanceof TimeoutException
+                  && retry
+                      < this.owningClient.getParameterProvider().getBlobUploadMaxRetryCount()) {
                 logger.logInfo(
-                    "Retry on waiting for uploading blob={}", futureBlob.getKey().getFilePath());
+                    "Retry on waiting for uploading blob={}, idx={}",
+                    futureBlob.getKey().getFilePath(),
+                    idx);
                 retry++;
                 break;
               }
@@ -178,11 +191,12 @@ List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyT
 
           if (blobs.size() > 0 && !isTestMode) {
             logger.logInfo(
-                "Start to registering blobs in client={}, totalBlobListSize={},"
-                    + " currentBlobListSize={}",
+                "Start registering blobs in client={}, totalBlobListSize={},"
+                    + " currentBlobListSize={}, idx={}",
                 this.owningClient.getName(),
                 oldList.size(),
-                blobs.size());
+                blobs.size(),
+                idx);
             Timer.Context registerContext =
                 Utils.createTimerContext(this.owningClient.registerLatency);
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 89df6231a..ce4f7e865 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -5,6 +5,7 @@
 package net.snowflake.ingest.streaming.internal;
 
 import static net.snowflake.ingest.utils.Constants.INSERT_THROTTLE_MAX_RETRY_COUNT;
+import static net.snowflake.ingest.utils.Constants.LOW_RUNTIME_MEMORY_THRESHOLD_IN_BYTES;
 import static net.snowflake.ingest.utils.Constants.MAX_CHUNK_SIZE_IN_BYTES;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
 
@@ -21,6 +22,7 @@
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.Utils;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.VectorSchemaRoot;
@@ -426,43 +428,40 @@ public String getLatestCommittedOffsetToken() {
     return response.getPersistedOffsetToken();
   }
 
-  /**
-   * Check whether we need to throttle the insert API under the following low memory condition:
-   * <li>system free_memory/total_memory < INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE
-   */
+  /** Check whether we need to throttle the insertRows API */
   void throttleInsertIfNeeded(Runtime runtime) {
-    int insertThrottleThresholdInPercentage =
-        this.owningClient.getParameterProvider().getInsertThrottleThresholdInPercentage();
-    if (runtime.freeMemory() * 100 / runtime.totalMemory() < insertThrottleThresholdInPercentage) {
-      long oldTotalMem = runtime.totalMemory();
-      long oldFreeMem = runtime.freeMemory();
-      int retry = 0;
-
-      long insertThrottleIntervalInMs =
-          this.owningClient.getParameterProvider().getInsertThrottleIntervalInMs();
-      while (runtime.freeMemory() * 100 / runtime.totalMemory()
-              < insertThrottleThresholdInPercentage
-          && retry < INSERT_THROTTLE_MAX_RETRY_COUNT) {
-        try {
-          Thread.sleep(insertThrottleIntervalInMs);
-          retry++;
-        } catch (InterruptedException e) {
-          throw new SFException(ErrorCode.INTERNAL_ERROR, "Insert throttle get interrupted");
-        }
+    int retry = 0;
+    long insertThrottleIntervalInMs =
+        this.owningClient.getParameterProvider().getInsertThrottleIntervalInMs();
+    while ((hasLowRuntimeMemory(runtime)
+            || (this.owningClient.getFlushService() != null
+                && this.owningClient.getFlushService().throttleDueToQueuedFlushTasks()))
+        && retry < INSERT_THROTTLE_MAX_RETRY_COUNT) {
+      try {
+        Thread.sleep(insertThrottleIntervalInMs);
+        retry++;
+      } catch (InterruptedException e) {
+        throw new SFException(ErrorCode.INTERNAL_ERROR, "Insert throttle get interrupted");
       }
+    }
+  }
 
+  /** Check whether we have a low runtime memory condition */
+  private boolean hasLowRuntimeMemory(Runtime runtime) {
+    int insertThrottleThresholdInPercentage =
+        this.owningClient.getParameterProvider().getInsertThrottleThresholdInPercentage();
+    boolean hasLowRuntimeMemory =
+        runtime.freeMemory() < LOW_RUNTIME_MEMORY_THRESHOLD_IN_BYTES
+            && runtime.freeMemory() * 100 / runtime.totalMemory()
+                < insertThrottleThresholdInPercentage;
+    if (hasLowRuntimeMemory) {
       logger.logWarn(
-          "InsertRows throttled due to JVM memory pressure, channel={}, timeInMs={}, max memory={},"
-              + " old total memory={}, old free memory={}, new total memory={}, new free"
-              + " memory={}.",
-          getFullyQualifiedName(),
-          retry * insertThrottleIntervalInMs,
-          runtime.maxMemory(),
-          oldTotalMem,
-          oldFreeMem,
-          runtime.totalMemory(),
-          runtime.freeMemory());
+          "Throttled due to memory pressure, client={}, channel={}.",
+          this.owningClient.getName(),
+          getFullyQualifiedName());
+      Utils.showMemory();
     }
+    return hasLowRuntimeMemory;
   }
 
   /** Check whether the channel is still valid, cleanup and throw an error if not */
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index 4d255e202..fed4022e1 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -31,8 +31,7 @@ public class Constants {
   public static final long RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL =
       7L; // Don't change, should match server side
   public static final int BLOB_UPLOAD_TIMEOUT_IN_SEC = 5;
-  public static final int BLOB_UPLOAD_MAX_RETRY_COUNT = 12;
-  public static final int INSERT_THROTTLE_MAX_RETRY_COUNT = 10;
+  public static final int INSERT_THROTTLE_MAX_RETRY_COUNT = 60;
   public static final long MAX_BLOB_SIZE_IN_BYTES = 512000000L;
   public static final long MAX_CHUNK_SIZE_IN_BYTES = 32000000L;
   public static final int BLOB_TAG_SIZE_IN_BYTES = 4;
@@ -43,7 +42,6 @@ public class Constants {
   public static final long THREAD_SHUTDOWN_TIMEOUT_IN_SEC = 300L;
   public static final String BLOB_EXTENSION_TYPE = "bdec";
   public static final int MAX_THREAD_COUNT = Integer.MAX_VALUE;
-  public static final int CPU_IO_TIME_RATIO = 1;
   public static final String CLIENT_CONFIGURE_ENDPOINT = "/v1/streaming/client/configure/";
   public static final int COMMIT_MAX_RETRY_COUNT = 60;
   public static final int COMMIT_RETRY_INTERVAL_IN_MS = 1000;
@@ -51,6 +49,7 @@ public class Constants {
   public static final int ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES = 16;
   public static final int MAX_STREAMING_INGEST_API_CHANNEL_RETRY = 3;
   public static final int STREAMING_INGEST_TELEMETRY_UPLOAD_INTERVAL_IN_SEC = 10;
+  public static final int LOW_RUNTIME_MEMORY_THRESHOLD_IN_BYTES = 100 * 1024 * 1024;
 
   // Channel level constants
   public static final String CHANNEL_STATUS_ENDPOINT = "/v1/streaming/channels/status/";
diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
index 3289df694..1ffc25eab 100644
--- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
+++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -6,27 +6,30 @@
 
 /** Utility class to provide configurable constants */
 public class ParameterProvider {
-  public static final String BUFFER_FLUSH_INTERVAL_IN_MILLIS_MAP_KEY =
+  public static final String BUFFER_FLUSH_INTERVAL_IN_MILLIS =
       "STREAMING_INGEST_CLIENT_SDK_BUFFER_FLUSH_INTERVAL_IN_MILLIS".toLowerCase();
-  public static final String BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_MAP_KEY =
+  public static final String BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS =
       "STREAMING_INGEST_CLIENT_SDK_BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS".toLowerCase();
-  public static final String INSERT_THROTTLE_INTERVAL_IN_MILLIS_MAP_KEY =
+  public static final String INSERT_THROTTLE_INTERVAL_IN_MILLIS =
       "STREAMING_INGEST_CLIENT_SDK_INSERT_THROTTLE_INTERVAL_IN_MILLIS".toLowerCase();
-  public static final String INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_MAP_KEY =
+  public static final String INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE =
       "STREAMING_INGEST_CLIENT_SDK_INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE".toLowerCase();
-  public static final String ENABLE_SNOWPIPE_STREAMING_METRICS_MAP_KEY =
+  public static final String ENABLE_SNOWPIPE_STREAMING_METRICS =
       "ENABLE_SNOWPIPE_STREAMING_JMX_METRICS".toLowerCase();
-
   public static final String BLOB_FORMAT_VERSION = "BLOB_FORMAT_VERSION".toLowerCase();
+  public static final String IO_TIME_CPU_RATIO = "IO_TIME_CPU_RATIO".toLowerCase();
+  public static final String BLOB_UPLOAD_MAX_RETRY_COUNT =
+      "BLOB_UPLOAD_MAX_RETRY_COUNT".toLowerCase();
 
   // Default values
   public static final long BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT = 1000;
   public static final long BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_DEFAULT = 100;
-  public static final long INSERT_THROTTLE_INTERVAL_IN_MILLIS_DEFAULT = 500;
-  public static final long INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_DEFAULT = 10;
+  public static final long INSERT_THROTTLE_INTERVAL_IN_MILLIS_DEFAULT = 1000;
+  public static final int INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_DEFAULT = 10;
   public static final boolean SNOWPIPE_STREAMING_METRICS_DEFAULT = false;
-
   public static final Constants.BdecVersion BLOB_FORMAT_VERSION_DEFAULT = Constants.BdecVersion.ONE;
+  public static final int IO_TIME_CPU_RATIO_DEFAULT = 2;
+  public static final int BLOB_UPLOAD_MAX_RETRY_COUNT_DEFAULT = 24;
 
   /** Map of parameter name to parameter value. This will be set by client/configure API Call. */
   private final Map<String, Object> parameterMap = new HashMap<>();
@@ -66,75 +69,101 @@ private void updateValue(
    */
   private void setParameterMap(Map<String, Object> parameterOverrides, Properties props) {
     this.updateValue(
-        BUFFER_FLUSH_INTERVAL_IN_MILLIS_MAP_KEY,
+        BUFFER_FLUSH_INTERVAL_IN_MILLIS,
         BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT,
         parameterOverrides,
         props);
 
     this.updateValue(
-        BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_MAP_KEY,
+        BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS,
         BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_DEFAULT,
         parameterOverrides,
         props);
 
     this.updateValue(
-        INSERT_THROTTLE_INTERVAL_IN_MILLIS_MAP_KEY,
+        INSERT_THROTTLE_INTERVAL_IN_MILLIS,
         INSERT_THROTTLE_INTERVAL_IN_MILLIS_DEFAULT,
         parameterOverrides,
         props);
 
     this.updateValue(
-        INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_MAP_KEY,
+        INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE,
         INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_DEFAULT,
         parameterOverrides,
         props);
 
     this.updateValue(
-        ENABLE_SNOWPIPE_STREAMING_METRICS_MAP_KEY,
+        ENABLE_SNOWPIPE_STREAMING_METRICS,
         SNOWPIPE_STREAMING_METRICS_DEFAULT,
         parameterOverrides,
         props);
 
     this.updateValue(BLOB_FORMAT_VERSION, BLOB_FORMAT_VERSION_DEFAULT, parameterOverrides, props);
     getBlobFormatVersion(); // to verify parsing the configured value
+
+    this.updateValue(IO_TIME_CPU_RATIO, IO_TIME_CPU_RATIO_DEFAULT, parameterOverrides, props);
+
+    this.updateValue(
+        BLOB_UPLOAD_MAX_RETRY_COUNT,
+        BLOB_UPLOAD_MAX_RETRY_COUNT_DEFAULT,
+        parameterOverrides,
+        props);
   }
 
   /** @return Longest interval in milliseconds between buffer flushes */
   public long getBufferFlushIntervalInMs() {
-    return (long)
+    Object val =
         this.parameterMap.getOrDefault(
-            BUFFER_FLUSH_INTERVAL_IN_MILLIS_MAP_KEY, BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT);
+            BUFFER_FLUSH_INTERVAL_IN_MILLIS, BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT);
+    if (val instanceof String) {
+      return Long.parseLong(val.toString());
+    }
+    return (long) val;
   }
 
   /** @return Time in milliseconds between checks to see if the buffer should be flushed */
   public long getBufferFlushCheckIntervalInMs() {
-    return (long)
+    Object val =
         this.parameterMap.getOrDefault(
-            BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_MAP_KEY,
-            BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_DEFAULT);
+            BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_DEFAULT);
+    if (val instanceof String) {
+      return Long.parseLong(val.toString());
+    }
+    return (long) val;
   }
 
   /** @return Duration in milliseconds to delay data insertion to the buffer when throttled */
   public long getInsertThrottleIntervalInMs() {
-    return (long)
+    Object val =
         this.parameterMap.getOrDefault(
-            INSERT_THROTTLE_INTERVAL_IN_MILLIS_MAP_KEY, INSERT_THROTTLE_INTERVAL_IN_MILLIS_DEFAULT);
+            INSERT_THROTTLE_INTERVAL_IN_MILLIS, INSERT_THROTTLE_INTERVAL_IN_MILLIS_DEFAULT);
+    if (val instanceof String) {
+      return Long.parseLong(val.toString());
+    }
+    return (long) val;
   }
 
   /** @return Percent of free total memory at which we throttle row inserts */
   public int getInsertThrottleThresholdInPercentage() {
-    return ((Long)
-            this.parameterMap.getOrDefault(
-                INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_MAP_KEY,
-                INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_DEFAULT))
-        .intValue();
+    Object val =
+        this.parameterMap.getOrDefault(
+            INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE,
+            INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_DEFAULT);
+    if (val instanceof String) {
+      return Integer.parseInt(val.toString());
+    }
+    return (int) val;
   }
 
   /** @return true if jmx metrics are enabled for a client */
   public boolean hasEnabledSnowpipeStreamingMetrics() {
-    return (Boolean)
+    Object val =
         this.parameterMap.getOrDefault(
-            ENABLE_SNOWPIPE_STREAMING_METRICS_MAP_KEY, SNOWPIPE_STREAMING_METRICS_DEFAULT);
+            ENABLE_SNOWPIPE_STREAMING_METRICS, SNOWPIPE_STREAMING_METRICS_DEFAULT);
+    if (val instanceof String) {
+      return Boolean.parseBoolean(val.toString());
+    }
+    return (boolean) val;
   }
 
   /** @return Blob format version: 1 (arrow stream write mode), 2 (arrow file write mode) etc */
@@ -154,6 +183,29 @@ public Constants.BdecVersion getBlobFormatVersion() {
     return Constants.BdecVersion.fromInt((int) val);
   }
 
+  /**
+   * @return the IO_TIME/CPU ratio that we will use to determine the number of buildAndUpload
+   *     threads
+   */
+  public int getIOTimeCpuRatio() {
+    Object val = this.parameterMap.getOrDefault(IO_TIME_CPU_RATIO, IO_TIME_CPU_RATIO_DEFAULT);
+    if (val instanceof String) {
+      return Integer.parseInt(val.toString());
+    }
+    return (int) val;
+  }
+
+  /** @return the max retry count when waiting for a blob upload task to finish */
+  public int getBlobUploadMaxRetryCount() {
+    Object val =
+        this.parameterMap.getOrDefault(
+            BLOB_UPLOAD_MAX_RETRY_COUNT, BLOB_UPLOAD_MAX_RETRY_COUNT_DEFAULT);
+    if (val instanceof String) {
+      return Integer.parseInt(val.toString());
+    }
+    return (int) val;
+  }
+
   @Override
   public String toString() {
     return "ParameterProvider{" + "parameterMap=" + parameterMap + '}';
diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java
index 68ef2124d..e37a08441 100644
--- a/src/main/java/net/snowflake/ingest/utils/Utils.java
+++ b/src/main/java/net/snowflake/ingest/utils/Utils.java
@@ -8,7 +8,11 @@
 import static net.snowflake.ingest.utils.Constants.USER;
 
 import com.codahale.metrics.Timer;
+import io.netty.util.internal.PlatformDependent;
 import java.io.StringReader;
+import java.lang.management.BufferPoolMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryUsage;
 import java.security.KeyFactory;
 import java.security.KeyPair;
 import java.security.NoSuchAlgorithmException;
@@ -19,6 +23,7 @@
 import java.security.spec.InvalidKeySpecException;
 import java.security.spec.PKCS8EncodedKeySpec;
 import java.security.spec.RSAPublicKeySpec;
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
@@ -260,4 +265,40 @@ public static void closeAllocator(BufferAllocator alloc) {
     }
     alloc.close();
   }
+
+  /** Util function to show memory usage info and debug memory issue in the SDK */
+  public static void showMemory() {
+    List<BufferPoolMXBean> pools = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class);
+    for (BufferPoolMXBean pool : pools) {
+      logger.logInfo(
+          "Pool name={}, pool count={}, memory used={}, total capacity={}",
+          pool.getName(),
+          pool.getCount(),
+          pool.getMemoryUsed(),
+          pool.getTotalCapacity());
+    }
+
+    Runtime runtime = Runtime.getRuntime();
+    logger.logInfo(
+        "Max direct memory={}, max runtime memory={}, total runtime memory={}, free runtime"
+            + " memory={}",
+        PlatformDependent.maxDirectMemory(),
+        runtime.maxMemory(),
+        runtime.totalMemory(),
+        runtime.freeMemory());
+
+    MemoryUsage nonHeapMem = ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage();
+    logger.logInfo(
+        "Non-heap memory usage max={}, used={}, committed={}",
+        nonHeapMem.getMax(),
+        nonHeapMem.getUsed(),
+        nonHeapMem.getCommitted());
+
+    MemoryUsage heapMem = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
+    logger.logInfo(
+        "Heap memory usage max={}, used={}, committed={}",
+        heapMem.getMax(),
+        heapMem.getUsed(),
+        heapMem.getCommitted());
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
index 292ef36d2..7a506a1d1 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
@@ -13,24 +13,28 @@ public class ParameterProviderTest {
   public void withValuesSet() {
     Properties prop = new Properties();
     Map<String, Object> parameterMap = new HashMap<>();
-    parameterMap.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS_MAP_KEY, 3L);
-    parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_MAP_KEY, 4L);
-    parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_MAP_KEY, 6L);
-    parameterMap.put(ParameterProvider.INSERT_THROTTLE_INTERVAL_IN_MILLIS_MAP_KEY, 7L);
+    parameterMap.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS, 3L);
+    parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 4L);
+    parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE, 6);
+    parameterMap.put(ParameterProvider.INSERT_THROTTLE_INTERVAL_IN_MILLIS, 7L);
+    parameterMap.put(ParameterProvider.IO_TIME_CPU_RATIO, 10);
+    parameterMap.put(ParameterProvider.BLOB_UPLOAD_MAX_RETRY_COUNT, 100);
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
 
     Assert.assertEquals(3, parameterProvider.getBufferFlushIntervalInMs());
     Assert.assertEquals(4, parameterProvider.getBufferFlushCheckIntervalInMs());
     Assert.assertEquals(6, parameterProvider.getInsertThrottleThresholdInPercentage());
     Assert.assertEquals(7, parameterProvider.getInsertThrottleIntervalInMs());
+    Assert.assertEquals(10, parameterProvider.getIOTimeCpuRatio());
+    Assert.assertEquals(100, parameterProvider.getBlobUploadMaxRetryCount());
   }
 
   @Test
   public void withNullProps() {
     Map<String, Object> parameterMap = new HashMap<>();
-    parameterMap.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS_MAP_KEY, 3L);
-    parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_MAP_KEY, 4L);
-    parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_MAP_KEY, 6L);
+    parameterMap.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS, 3L);
+    parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 4L);
+    parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE, 6);
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, null);
 
     Assert.assertEquals(3, parameterProvider.getBufferFlushIntervalInMs());
@@ -44,9 +48,9 @@ public void withNullProps() {
   @Test
   public void withNullParameterMap() {
     Properties props = new Properties();
-    props.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS_MAP_KEY, 3L);
-    props.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_MAP_KEY, 4L);
-    props.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_MAP_KEY, 6L);
+    props.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS, 3L);
+    props.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 4L);
+    props.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE, 6);
     ParameterProvider parameterProvider = new ParameterProvider(null, props);
 
     Assert.assertEquals(3, parameterProvider.getBufferFlushIntervalInMs());
@@ -91,5 +95,10 @@ public void withDefaultValues() {
     Assert.assertEquals(
         ParameterProvider.INSERT_THROTTLE_INTERVAL_IN_MILLIS_DEFAULT,
         parameterProvider.getInsertThrottleIntervalInMs());
+    Assert.assertEquals(
+        ParameterProvider.IO_TIME_CPU_RATIO_DEFAULT, parameterProvider.getIOTimeCpuRatio());
+    Assert.assertEquals(
+        ParameterProvider.BLOB_UPLOAD_MAX_RETRY_COUNT_DEFAULT,
+        parameterProvider.getBlobUploadMaxRetryCount());
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index becbde047..ab48cc384 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -10,7 +10,7 @@
 import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
 import static net.snowflake.ingest.utils.Constants.ROLE;
 import static net.snowflake.ingest.utils.Constants.USER;
-import static net.snowflake.ingest.utils.ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS_MAP_KEY;
+import static net.snowflake.ingest.utils.ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.PropertyAccessor;
@@ -146,10 +146,10 @@ public void testConstructorParameters() throws Exception {
     prop.put(ACCOUNT_URL, TestUtils.getHost());
     prop.put(PRIVATE_KEY, TestUtils.getPrivateKey());
     prop.put(ROLE, "role");
-    prop.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS_MAP_KEY, 123);
+    prop.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS, 123);
 
     Map<String, Object> parameterMap = new HashMap<>();
-    parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_MAP_KEY, 321);
+    parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 321);
 
     SnowflakeStreamingIngestClientInternal<?> client =
         (SnowflakeStreamingIngestClientInternal<?>)
@@ -181,7 +181,7 @@ public void testClientFactoryWithJmxMetrics() throws Exception {
             SnowflakeStreamingIngestClientFactory.builder("client")
                 .setProperties(prop)
                 .setParameterOverrides(
-                    Collections.singletonMap(ENABLE_SNOWPIPE_STREAMING_METRICS_MAP_KEY, true))
+                    Collections.singletonMap(ENABLE_SNOWPIPE_STREAMING_METRICS, true))
                 .build();
 
     Assert.assertEquals("client", client.getName());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 6b49fbf29..175db5fb8 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -161,11 +161,13 @@ public void testSimpleIngest() throws Exception {
   @Test
   public void testParameterOverrides() throws Exception {
     Map<String, Object> parameterMap = new HashMap<>();
-    parameterMap.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS_MAP_KEY, 30L);
-    parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_MAP_KEY, 50L);
-    parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_MAP_KEY, 1L);
-    parameterMap.put(ParameterProvider.INSERT_THROTTLE_INTERVAL_IN_MILLIS_MAP_KEY, 1L);
-    parameterMap.put(ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS_MAP_KEY, true);
+    parameterMap.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS, 30L);
+    parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 50L);
+    parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE, 1);
+    parameterMap.put(ParameterProvider.INSERT_THROTTLE_INTERVAL_IN_MILLIS, 1L);
+    parameterMap.put(ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS, true);
+    parameterMap.put(ParameterProvider.IO_TIME_CPU_RATIO, 1);
+    parameterMap.put(ParameterProvider.BLOB_UPLOAD_MAX_RETRY_COUNT, 1);
     client =
         (SnowflakeStreamingIngestClientInternal<?>)
             SnowflakeStreamingIngestClientFactory.builder("testParameterOverridesClient")

From a6f566fdc7dee385c799852aba5d444d7cc256a5 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 29 Sep 2022 12:25:25 -0700
Subject: [PATCH 067/356] NO-SNOW: add stacktrace during blob building failure
 (#232)

This PR contains two changes:

1. As part of investigating an encryption related issue, we realize the stacktrace and error message are not captured in log for any encryption related exceptions, so this PR adds that support
2. Add a small message indicates the code path of channel invalidation
3. Improve exception handling
---
 .../streaming/internal/ChannelCache.java      |  4 +-
 .../streaming/internal/FlushService.java      | 58 ++++++++++++++-----
 .../streaming/internal/RegisterService.java   | 13 ++---
 ...owflakeStreamingIngestChannelInternal.java |  7 ++-
 .../net/snowflake/ingest/utils/Utils.java     | 13 +++++
 .../SnowflakeStreamingIngestChannelTest.java  |  2 +-
 6 files changed, 68 insertions(+), 29 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
index 42781deec..35ca580bc 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
@@ -37,7 +37,7 @@ void addChannel(SnowflakeStreamingIngestChannelInternal<T> channel) {
         channels.put(channel.getName(), channel);
     // Invalidate old channel if it exits to block new inserts and return error to users earlier
     if (oldChannel != null) {
-      oldChannel.invalidate();
+      oldChannel.invalidate("removed from cache");
     }
   }
 
@@ -89,7 +89,7 @@ void invalidateChannelIfSequencersMatch(
     if (channelsMapPerTable != null) {
       SnowflakeStreamingIngestChannelInternal<T> channel = channelsMapPerTable.get(channelName);
       if (channel != null && channel.getChannelSequencer().equals(channelSequencer)) {
-        channel.invalidate();
+        channel.invalidate("invalidate with matched sequencer");
       }
     }
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 8a4d83df0..df5522469 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -9,6 +9,7 @@
 import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
 import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
 import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
 import com.codahale.metrics.Timer;
 import java.io.ByteArrayOutputStream;
@@ -279,7 +280,25 @@ private void createWorkers() {
     this.flushWorker = Executors.newSingleThreadScheduledExecutor(flushThreadFactory);
     this.flushWorker.scheduleWithFixedDelay(
         () -> {
-          flush(false);
+          flush(false)
+              .exceptionally(
+                  e -> {
+                    String errorMessage =
+                        String.format(
+                            "Background flush task failed, client=%s, exception=%s, detail=%s,"
+                                + " trace=%s.",
+                            this.owningClient.getName(),
+                            e.getCause(),
+                            e.getCause().getMessage(),
+                            getStackTrace(e.getCause()));
+                    logger.logError(errorMessage);
+                    if (this.owningClient.getTelemetryService() != null) {
+                      this.owningClient
+                          .getTelemetryService()
+                          .reportClientFailure(this.getClass().getSimpleName(), errorMessage);
+                    }
+                    return null;
+                  });
         },
         0,
         this.owningClient.getParameterProvider().getBufferFlushCheckIntervalInMs(),
@@ -357,28 +376,39 @@ void distributeFlushTasks() {
                     () -> {
                       try {
                         return buildAndUpload(filePath, blobData);
-                      } catch (IOException e) {
+                      } catch (Throwable e) {
+                        Throwable ex = e.getCause() == null ? e : e.getCause();
                         String errorMessage =
                             String.format(
                                 "Building blob failed, client=%s, file=%s, exception=%s,"
-                                    + " detail=%s, all channels in the blob will be invalidated",
-                                this.owningClient.getName(), filePath, e, e.getMessage());
+                                    + " detail=%s, trace=%s, all channels in the blob will be"
+                                    + " invalidated",
+                                this.owningClient.getName(),
+                                filePath,
+                                ex,
+                                ex.getMessage(),
+                                getStackTrace(ex));
                         logger.logError(errorMessage);
                         if (this.owningClient.getTelemetryService() != null) {
                           this.owningClient
                               .getTelemetryService()
                               .reportClientFailure(this.getClass().getSimpleName(), errorMessage);
                         }
-                        invalidateAllChannelsInBlob(blobData);
-                        return null;
-                      } catch (NoSuchAlgorithmException e) {
-                        throw new SFException(e, ErrorCode.MD5_HASHING_NOT_AVAILABLE);
-                      } catch (InvalidAlgorithmParameterException
-                          | NoSuchPaddingException
-                          | IllegalBlockSizeException
-                          | BadPaddingException
-                          | InvalidKeyException e) {
-                        throw new SFException(e, ErrorCode.ENCRYPTION_FAILURE);
+
+                        if (e instanceof IOException) {
+                          invalidateAllChannelsInBlob(blobData);
+                          return null;
+                        } else if (e instanceof NoSuchAlgorithmException) {
+                          throw new SFException(e, ErrorCode.MD5_HASHING_NOT_AVAILABLE);
+                        } else if (e instanceof InvalidAlgorithmParameterException
+                            | e instanceof NoSuchPaddingException
+                            | e instanceof IllegalBlockSizeException
+                            | e instanceof BadPaddingException
+                            | e instanceof InvalidKeyException) {
+                          throw new SFException(e, ErrorCode.ENCRYPTION_FAILURE);
+                        } else {
+                          throw new SFException(e, ErrorCode.INTERNAL_ERROR, e.getMessage());
+                        }
                       }
                     },
                     this.buildUploadWorkers)));
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index 4887441c0..739ae5499 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -5,6 +5,7 @@
 package net.snowflake.ingest.streaming.internal;
 
 import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
 import com.codahale.metrics.Timer;
 import java.util.ArrayList;
@@ -156,24 +157,18 @@ List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyT
                 retry++;
                 break;
               }
-              StringBuilder stackTrace = new StringBuilder();
-              if (e.getCause() != null) {
-                for (StackTraceElement element : e.getCause().getStackTrace()) {
-                  stackTrace.append(System.lineSeparator()).append(element.toString());
-                }
-              }
               String errorMessage =
                   String.format(
                       "Building or uploading blob failed, client=%s, file=%s, exception=%s,"
-                          + " detail=%s, cause=%s, detail=%s trace=%s all channels in the blob"
-                          + " will be invalidated",
+                          + " detail=%s, cause=%s, cause_detail=%s, cause_trace=%s all channels in"
+                          + " the blob will be invalidated",
                       this.owningClient.getName(),
                       futureBlob.getKey().getFilePath(),
                       e,
                       e.getMessage(),
                       e.getCause(),
                       e.getCause() == null ? null : e.getCause().getMessage(),
-                      stackTrace);
+                      getStackTrace(e.getCause()));
               logger.logError(errorMessage);
               if (this.owningClient.getTelemetryService() != null) {
                 this.owningClient
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index ce4f7e865..d2d64db29 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -244,14 +244,15 @@ public boolean isValid() {
   }
 
   /** Mark the channel as invalid, and release resources */
-  void invalidate() {
+  void invalidate(String message) {
     this.isValid = false;
     this.rowBuffer.close("invalidate");
     logger.logWarn(
-        "Channel is invalidated, name={}, channel sequencer={}, row sequencer={}",
+        "Channel is invalidated, name={}, channel sequencer={}, row sequencer={}, message={}",
         getFullyQualifiedName(),
         channelSequencer,
-        rowSequencer);
+        rowSequencer,
+        message);
   }
 
   /** @return a boolean to indicate whether the channel is closed or not */
diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java
index e37a08441..2c9dbb156 100644
--- a/src/main/java/net/snowflake/ingest/utils/Utils.java
+++ b/src/main/java/net/snowflake/ingest/utils/Utils.java
@@ -301,4 +301,17 @@ public static void showMemory() {
         heapMem.getUsed(),
         heapMem.getCommitted());
   }
+
+  /** Return the stack trace for a given exception */
+  public static String getStackTrace(Throwable e) {
+    if (e == null) {
+      return null;
+    }
+
+    StringBuilder stackTrace = new StringBuilder();
+    for (StackTraceElement element : e.getStackTrace()) {
+      stackTrace.append(System.lineSeparator()).append(element.toString());
+    }
+    return stackTrace.toString();
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index a7710932d..5c341e4c9 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -141,7 +141,7 @@ public void testChannelValid() {
             OpenChannelRequest.OnErrorOption.CONTINUE);
 
     Assert.assertTrue(channel.isValid());
-    channel.invalidate();
+    channel.invalidate("from testChannelValid");
     Assert.assertFalse(channel.isValid());
 
     // Can't insert rows to invalid channel

From 31bfab2d8265127561613bb3b700d5a9c51696c2 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 29 Sep 2022 12:27:54 -0700
Subject: [PATCH 068/356] 1.0.2-beta.5 release

---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 9f377c7ca..e60d2d989 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
     <!-- Arifact name and version information -->
     <groupId>net.snowflake</groupId>
     <artifactId>snowflake-ingest-sdk</artifactId>
-    <version>1.0.2-beta.4</version>
+    <version>1.0.2-beta.5</version>
     <packaging>jar</packaging>
     <name>Snowflake Ingest SDK</name>
     <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 349c08ace..07085d582 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -127,7 +127,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.0.2-beta.4";
+  public static final String DEFAULT_VERSION = "1.0.2-beta.5";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From fac1d52f0f29da1c8ce9792210bb853e8b6da2ea Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Fri, 30 Sep 2022 19:04:38 +0200
Subject: [PATCH 069/356] SNOW-655614 Data type validation: Semi-structured
 types (#233)

SNOW-655614 Data type validation: Semi-structured types

More restricted validation of VARIANT, ARRAY AND OBJECT has been implemented:
* Only selected types are allowed
    * String MUST be a valid JSON, otherwise it is rejected
        * i.e. "foo" is rejected, but "\"foo\"" is allowed
    * java primitive data types and their arrays
    * java.time.* objects
    * T[] if T is a valid variant type
    * List<T> if T is a valid variant type
    * Map<String, T> if T is a valid variant type
* Custom JSON serialization of ZonedDateTime and byte[]
* Collections (array, list, map) are recursively validated
* ARRAY: Non-array input is converted into a 1-element array
* OBJECT: Non-object input is rejected
* Due to SNOW-664249, max allowed limit is reduced by a few bytes
---
 .../internal/DataValidationUtil.java          | 272 +++++++++---
 .../serialization/ByteArraySerializer.java    |  26 ++
 .../ZonedDateTimeSerializer.java              |  20 +
 .../internal/DataValidationUtilTest.java      | 401 +++++++++++++++---
 .../datatypes/AbstractDataTypeTest.java       |  14 +-
 .../streaming/internal/datatypes/NullIT.java  |  33 ++
 .../internal/datatypes/SemiStructuredIT.java  | 216 ++++++++++
 7 files changed, 857 insertions(+), 125 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/serialization/ByteArraySerializer.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/serialization/ZonedDateTimeSerializer.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 59cd21ba6..881a4c2f3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -7,10 +7,14 @@
 import static net.snowflake.client.jdbc.internal.snowflake.common.core.SnowflakeDateTimeFormat.DATE;
 import static net.snowflake.client.jdbc.internal.snowflake.common.core.SnowflakeDateTimeFormat.TIMESTAMP;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.LocalTime;
@@ -18,7 +22,9 @@
 import java.time.OffsetTime;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.TimeZone;
@@ -29,6 +35,8 @@
 import net.snowflake.client.jdbc.internal.snowflake.common.core.SFTimestamp;
 import net.snowflake.client.jdbc.internal.snowflake.common.core.SnowflakeDateTimeFormat;
 import net.snowflake.client.jdbc.internal.snowflake.common.util.Power10;
+import net.snowflake.ingest.streaming.internal.serialization.ByteArraySerializer;
+import net.snowflake.ingest.streaming.internal.serialization.ZonedDateTimeSerializer;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 
@@ -37,12 +45,32 @@ class DataValidationUtil {
   public static final int BYTES_8_MB = 8 * 1024 * 1024;
   public static final int BYTES_16_MB = 2 * BYTES_8_MB;
 
+  // TODO SNOW-664249: There is a few-byte mismatch between the value sent by the user and its
+  // server-side representation. Validation leaves a small buffer for this difference.
+  static final int MAX_SEMI_STRUCTURED_LENGTH = BYTES_16_MB - 64;
+
   private static final TimeZone DEFAULT_TIMEZONE =
       TimeZone.getTimeZone("America/Los_Angeles"); // default value of TIMEZONE system parameter
   private static final TimeZone GMT = TimeZone.getTimeZone("GMT");
 
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
+  // The version of Jackson we are using does not support serialization of date objects from the
+  // java.time package. Here we define a module with custom java.time serializers. Additionally, we
+  // define custom serializer for byte[] because the Jackson default is to serialize it as
+  // base64-encoded string, and we would like to serialize it as JSON array of numbers.
+  static {
+    SimpleModule module = new SimpleModule();
+    module.addSerializer(byte[].class, new ByteArraySerializer());
+    module.addSerializer(ZonedDateTime.class, new ZonedDateTimeSerializer());
+    module.addSerializer(LocalTime.class, new ToStringSerializer());
+    module.addSerializer(OffsetTime.class, new ToStringSerializer());
+    module.addSerializer(LocalDate.class, new ToStringSerializer());
+    module.addSerializer(LocalDateTime.class, new ToStringSerializer());
+    module.addSerializer(OffsetDateTime.class, new ToStringSerializer());
+    objectMapper.registerModule(module);
+  }
+
   /**
    * Creates a new SnowflakeDateTimeFormat. In order to avoid SnowflakeDateTimeFormat's
    * synchronization blocks, we create a new instance when needed instead of sharing one instance.
@@ -52,80 +80,213 @@ private static SnowflakeDateTimeFormat createDateTimeFormatter() {
   }
 
   /**
-   * Expects string JSON
+   * Validates and parses input as JSON. All types in the object tree must be valid variant types,
+   * see {@link DataValidationUtil#isAllowedSemiStructuredType}.
    *
-   * @param input the input data, must be able to convert to String
+   * @param input Object to validate
+   * @return JSON tree representing the input
    */
-  static String validateAndParseVariant(Object input) {
-    String output;
-    try {
-      output = input.toString();
-    } catch (Exception e) {
-      throw new SFException(
-          e, ErrorCode.INVALID_ROW, input, "Input column can't be convert to String.");
+  private static JsonNode validateAndParseSemiStructuredAsJsonTree(
+      Object input, String snowflakeType) {
+    if (input instanceof String) {
+      try {
+        return objectMapper.readTree((String) input);
+      } catch (JsonProcessingException e) {
+        throw valueFormatNotAllowedException(input, snowflakeType, "Not a valid JSON");
+      }
+    } else if (isAllowedSemiStructuredType(input)) {
+      return objectMapper.valueToTree(input);
     }
 
-    if (output.length() > BYTES_16_MB) {
-      throw new SFException(
-          ErrorCode.INVALID_ROW,
-          input.toString(),
-          String.format("Variant too long: length=%d maxLength=%d", output.length(), BYTES_16_MB));
+    throw typeNotAllowedException(
+        input.getClass(),
+        snowflakeType,
+        new String[] {
+          "String",
+          "Primitive data types and their arrays",
+          "java.time.*",
+          "List<T>",
+          "Map<String, T>",
+          "T[]"
+        });
+  }
+
+  /**
+   * Validates and parses input as JSON. All types in the object tree must be valid variant types,
+   * see {@link DataValidationUtil#isAllowedSemiStructuredType}.
+   *
+   * @param input Object to validate
+   * @return JSON string representing the input
+   */
+  static String validateAndParseVariant(Object input) {
+    String output = validateAndParseSemiStructuredAsJsonTree(input, "VARIANT").toString();
+    int stringLength = output.getBytes(StandardCharsets.UTF_8).length;
+    if (stringLength > MAX_SEMI_STRUCTURED_LENGTH) {
+      throw valueFormatNotAllowedException(
+          input,
+          "VARIANT",
+          String.format(
+              "Variant too long: length=%d maxLength=%d",
+              stringLength, MAX_SEMI_STRUCTURED_LENGTH));
     }
     return output;
   }
 
   /**
-   * Expects an Array or List object
+   * Validates that passed object is allowed data type for semi-structured columns (i.e. VARIANT,
+   * ARRAY, OBJECT). For non-trivial types like maps, arrays or lists, it recursively traverses the
+   * object tree and validates that all types in the tree are also allowed. Allowed Java types:
    *
-   * @param input the input data, must be able to convert to String
+   * <ul>
+   *   <li>primitive types (int, long, boolean, ...)
+   *   <li>String
+   *   <li>BigInteger
+   *   <li>BigDecimal
+   *   <li>LocalTime
+   *   <li>OffsetTime
+   *   <li>LocalDate
+   *   <li>LocalDateTime
+   *   <li>OffsetDateTime
+   *   <li>ZonedDateTime
+   *   <li>Map<String, T> where T is an allowed semi-structured type
+   *   <li>List<T> where T is an allowed semi-structured type
+   *   <li>primitive arrays (char[], int[], ...)
+   *   <li>T[] where T is an allowed semi-structured type
+   * </ul>
+   *
+   * @param o Object to validate
+   * @return If the passed object is allowed for ingestion into semi-structured column
    */
-  static String validateAndParseArray(Object input) {
-    if (!input.getClass().isArray() && !(input instanceof List)) {
-      throw new SFException(
-          ErrorCode.INVALID_ROW, input, "Input column must be an Array or List object.");
+  static boolean isAllowedSemiStructuredType(Object o) {
+    // Allow null
+    if (o == null) {
+      return true;
     }
 
-    String output;
-    try {
-      output = objectMapper.writeValueAsString(input);
-    } catch (Exception e) {
-      throw new SFException(
-          e,
-          ErrorCode.INVALID_ROW,
-          input.toString(),
-          "Input column can't be convert to a valid string");
+    // Allow string
+    if (o instanceof String) {
+      return true;
+    }
+
+    // Allow all primitive Java data types
+    if (o instanceof Long
+        || o instanceof Integer
+        || o instanceof Short
+        || o instanceof Byte
+        || o instanceof Float
+        || o instanceof Double
+        || o instanceof Boolean
+        || o instanceof Character) {
+      return true;
     }
 
+    // Allow BigInteger and BigDecimal
+    if (o instanceof BigInteger || o instanceof BigDecimal) {
+      return true;
+    }
+
+    // Allow supported types from java.time package
+    if (o instanceof java.time.LocalTime
+        || o instanceof OffsetTime
+        || o instanceof LocalDate
+        || o instanceof LocalDateTime
+        || o instanceof ZonedDateTime
+        || o instanceof OffsetDateTime) {
+      return true;
+    }
+
+    // Map<String, T> is allowed, as long as T is also a supported semi-structured type
+    if (o instanceof Map) {
+      boolean allKeysAreStrings =
+          ((Map<?, ?>) o).keySet().stream().allMatch(x -> x instanceof String);
+      if (!allKeysAreStrings) {
+        return false;
+      }
+      boolean allValuesAreAllowed =
+          ((Map<?, ?>) o)
+              .values().stream().allMatch(DataValidationUtil::isAllowedSemiStructuredType);
+      return allValuesAreAllowed;
+    }
+
+    // Allow arrays of primitive data types
+    if (o instanceof byte[]
+        || o instanceof short[]
+        || o instanceof int[]
+        || o instanceof long[]
+        || o instanceof float[]
+        || o instanceof double[]
+        || o instanceof boolean[]
+        || o instanceof char[]) {
+      return true;
+    }
+
+    // Allow arrays of allowed semi-structured objects
+    if (o.getClass().isArray()) {
+      return Arrays.stream((Object[]) o).allMatch(DataValidationUtil::isAllowedSemiStructuredType);
+    }
+
+    // Allow lists consisting of allowed semi-structured objects
+    if (o instanceof List) {
+      return ((List<?>) o).stream().allMatch(DataValidationUtil::isAllowedSemiStructuredType);
+    }
+
+    // If nothing matches, reject the input
+    return false;
+  }
+
+  /**
+   * Validates and parses JSON array. Non-array types are converted into single-element arrays. All
+   * types in the array tree must be valid variant types, see {@link
+   * DataValidationUtil#isAllowedSemiStructuredType}.
+   *
+   * @param input Object to validate
+   * @return JSON array representing the input
+   */
+  static String validateAndParseArray(Object input) {
+    JsonNode jsonNode = validateAndParseSemiStructuredAsJsonTree(input, "ARRAY");
+
+    // Non-array values are ingested as single-element arrays, mimicking the Worksheets behavior
+    if (!jsonNode.isArray()) {
+      jsonNode = objectMapper.createArrayNode().add(jsonNode);
+    }
+
+    String output = jsonNode.toString();
     // Throw an exception if the size is too large
-    if (output.length() > BYTES_16_MB) {
-      throw new SFException(
-          ErrorCode.INVALID_ROW,
-          input.toString(),
-          String.format("Array too large. length=%d maxLength=%d", output.length(), BYTES_16_MB));
+    int stringLength = output.getBytes(StandardCharsets.UTF_8).length;
+    if (stringLength > MAX_SEMI_STRUCTURED_LENGTH) {
+      throw valueFormatNotAllowedException(
+          jsonNode.toString(),
+          "ARRAY",
+          String.format(
+              "Array too large. length=%d maxLength=%d", stringLength, MAX_SEMI_STRUCTURED_LENGTH));
     }
     return output;
   }
 
   /**
-   * Expects string JSON or JsonNode
+   * Validates and parses JSON object. Input is rejected if the value does not represent JSON object
+   * (e.g. String '{}' or Map<String, T>). All types in the object tree must be valid variant types,
+   * see {@link DataValidationUtil#isAllowedSemiStructuredType}.
    *
-   * @param input Must be valid string JSON or JsonNode
+   * @param input Object to validate
+   * @return JSON object representing the input
    */
   static String validateAndParseObject(Object input) {
-    String output;
-    try {
-      JsonNode node = objectMapper.readTree(input.toString());
-      output = node.toString();
-    } catch (Exception e) {
-      throw new SFException(
-          e, ErrorCode.INVALID_ROW, input.toString(), "Input column can't be convert to Json");
+    JsonNode jsonNode = validateAndParseSemiStructuredAsJsonTree(input, "OBJECT");
+    if (!jsonNode.isObject()) {
+      throw valueFormatNotAllowedException(jsonNode, "OBJECT", "Not an object");
     }
 
-    if (output.length() > BYTES_16_MB) {
-      throw new SFException(
-          ErrorCode.INVALID_ROW,
-          input.toString(),
-          String.format("Object too large. length=%d maxLength=%d", output.length(), BYTES_16_MB));
+    String output = jsonNode.toString();
+    // Throw an exception if the size is too large
+    int stringLength = output.getBytes(StandardCharsets.UTF_8).length;
+    if (stringLength > MAX_SEMI_STRUCTURED_LENGTH) {
+      throw valueFormatNotAllowedException(
+          output,
+          "OBJECT",
+          String.format(
+              "Object too large. length=%d maxLength=%d",
+              stringLength, MAX_SEMI_STRUCTURED_LENGTH));
     }
     return output;
   }
@@ -420,7 +581,7 @@ static byte[] validateAndParseBinary(Object input, Optional<Integer> maxLengthOp
 
   /**
    * Returns the number of units since 00:00, depending on the scale (scale=0: seconds, scale=3:
-   * milliseconds, scale=9: nanoseconds. Allowed Java types:
+   * milliseconds, scale=9: nanoseconds). Allowed Java types:
    *
    * <ul>
    *   <li>String
@@ -460,19 +621,6 @@ static BigInteger validateAndParseTime(Object input, int scale) {
     }
   }
 
-  static String getStringValue(Object value) {
-    String stringValue;
-
-    if (value instanceof String) {
-      stringValue = (String) value;
-    } else if (value instanceof BigDecimal) {
-      stringValue = ((BigDecimal) value).toPlainString();
-    } else {
-      stringValue = value.toString();
-    }
-    return stringValue;
-  }
-
   /**
    * Converts input to double value. Allowed Java types:
    *
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/serialization/ByteArraySerializer.java b/src/main/java/net/snowflake/ingest/streaming/internal/serialization/ByteArraySerializer.java
new file mode 100644
index 000000000..036b9a509
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/serialization/ByteArraySerializer.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal.serialization;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import java.io.IOException;
+
+/**
+ * Serialize Java byte arrays as JSON arrays of numbers instead of the default Jackson
+ * base64-encoding.
+ */
+public class ByteArraySerializer extends JsonSerializer<byte[]> {
+  @Override
+  public void serialize(byte[] value, JsonGenerator gen, SerializerProvider serializers)
+      throws IOException {
+    gen.writeStartArray();
+    for (byte v : value) {
+      gen.writeNumber(v);
+    }
+    gen.writeEndArray();
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/serialization/ZonedDateTimeSerializer.java b/src/main/java/net/snowflake/ingest/streaming/internal/serialization/ZonedDateTimeSerializer.java
new file mode 100644
index 000000000..3760362e8
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/serialization/ZonedDateTimeSerializer.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal.serialization;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import java.io.IOException;
+import java.time.ZonedDateTime;
+
+/** Snowflake does not support parsing zones, so serialize it in offset instead */
+public class ZonedDateTimeSerializer extends JsonSerializer<ZonedDateTime> {
+  @Override
+  public void serialize(ZonedDateTime value, JsonGenerator gen, SerializerProvider serializers)
+      throws IOException {
+    gen.writeString(value.toOffsetDateTime().toString());
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 27ecc992a..2d36de72c 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -2,19 +2,22 @@
 
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.BYTES_16_MB;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.BYTES_8_MB;
+import static net.snowflake.ingest.streaming.internal.DataValidationUtil.isAllowedSemiStructuredType;
+import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseArray;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseBigDecimal;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseBinary;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseBoolean;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseDate;
+import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseObject;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseReal;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseString;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseTime;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseTimestampNtzSb16;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseTimestampTz;
+import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseVariant;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 
-import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -24,8 +27,10 @@
 import java.time.LocalTime;
 import java.time.OffsetDateTime;
 import java.time.OffsetTime;
+import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
@@ -351,73 +356,113 @@ public void testValidateAndParseString() {
 
   @Test
   public void testValidateAndParseVariant() throws Exception {
+    assertEquals("1", validateAndParseVariant(1));
+    assertEquals("1", validateAndParseVariant("1"));
+    assertEquals("1", validateAndParseVariant("                          1   "));
     String stringVariant = "{\"key\":1}";
-    Assert.assertEquals(stringVariant, DataValidationUtil.validateAndParseVariant(stringVariant));
-    JsonNode nodeVariant = objectMapper.readTree(stringVariant);
-    Assert.assertEquals(stringVariant, DataValidationUtil.validateAndParseVariant(nodeVariant));
+    assertEquals(stringVariant, validateAndParseVariant(stringVariant));
 
-    char[] data = new char[20000000];
-    Arrays.fill(data, 'a');
-    String stringVal = new String(data);
-    try {
-      DataValidationUtil.validateAndParseVariant(stringVal);
-      Assert.fail("Expected INVALID_ROW error");
-    } catch (SFException err) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), err.getVendorCode());
-    }
+    // Test custom serializers
+    assertEquals(
+        "[-128,0,127]", validateAndParseVariant(new byte[] {Byte.MIN_VALUE, 0, Byte.MAX_VALUE}));
+    assertEquals(
+        "\"2022-09-28T03:04:12.123456789-07:00\"",
+        validateAndParseVariant(
+            ZonedDateTime.of(2022, 9, 28, 3, 4, 12, 123456789, ZoneId.of("America/Los_Angeles"))));
+
+    // Test forbidden values
+    expectError(
+        ErrorCode.INVALID_ROW,
+        DataValidationUtil::validateAndParseVariant,
+        objectMapper.readTree("{}"));
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseVariant, new Object());
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseVariant, "foo");
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseVariant, new Date());
+    expectError(
+        ErrorCode.INVALID_ROW,
+        DataValidationUtil::validateAndParseVariant,
+        Collections.singletonList(new Object()));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        DataValidationUtil::validateAndParseVariant,
+        Collections.singletonList(Collections.singletonMap("foo", new Object())));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        DataValidationUtil::validateAndParseVariant,
+        Collections.singletonMap(new Object(), "foo"));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        DataValidationUtil::validateAndParseVariant,
+        Collections.singletonMap("foo", new Object()));
   }
 
   @Test
   public void testValidateAndParseArray() throws Exception {
-    int invalidArray = 1;
-    try {
-      DataValidationUtil.validateAndParseArray(invalidArray);
-      Assert.fail("Expected INVALID_ROW error");
-    } catch (SFException err) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), err.getVendorCode());
-    }
-
+    assertEquals("[1]", validateAndParseArray(1));
+    assertEquals("[1]", validateAndParseArray("1"));
+    assertEquals("[1]", validateAndParseArray("                          1   "));
     int[] intArray = new int[] {1, 2, 3};
-    Assert.assertEquals("[1,2,3]", DataValidationUtil.validateAndParseArray(intArray));
+    assertEquals("[1,2,3]", validateAndParseArray(intArray));
 
     String[] stringArray = new String[] {"a", "b", "c"};
-    Assert.assertEquals(
-        "[\"a\",\"b\",\"c\"]", DataValidationUtil.validateAndParseArray(stringArray));
+    assertEquals("[\"a\",\"b\",\"c\"]", validateAndParseArray(stringArray));
 
     Object[] objectArray = new Object[] {1, 2, 3};
-    Assert.assertEquals("[1,2,3]", DataValidationUtil.validateAndParseArray(objectArray));
+    assertEquals("[1,2,3]", validateAndParseArray(objectArray));
 
     Object[] ObjectArrayWithNull = new Object[] {1, null, 3};
-    Assert.assertEquals(
-        "[1,null,3]", DataValidationUtil.validateAndParseArray(ObjectArrayWithNull));
+    assertEquals("[1,null,3]", validateAndParseArray(ObjectArrayWithNull));
 
     Object[][] nestedArray = new Object[][] {{1, 2, 3}, null, {4, 5, 6}};
-    Assert.assertEquals(
-        "[[1,2,3],null,[4,5,6]]", DataValidationUtil.validateAndParseArray(nestedArray));
+    assertEquals("[[1,2,3],null,[4,5,6]]", validateAndParseArray(nestedArray));
 
     List<Integer> intList = Arrays.asList(1, 2, 3);
-    Assert.assertEquals("[1,2,3]", DataValidationUtil.validateAndParseArray(intList));
+    assertEquals("[1,2,3]", validateAndParseArray(intList));
 
     List<Object> objectList = Arrays.asList(1, 2, 3);
-    Assert.assertEquals("[1,2,3]", DataValidationUtil.validateAndParseArray(objectList));
+    assertEquals("[1,2,3]", validateAndParseArray(objectList));
 
     List<Object> nestedList = Arrays.asList(Arrays.asList(1, 2, 3), 2, 3);
-    Assert.assertEquals("[[1,2,3],2,3]", DataValidationUtil.validateAndParseArray(nestedList));
+    assertEquals("[[1,2,3],2,3]", validateAndParseArray(nestedList));
+
+    // Test forbidden values
+    expectError(
+        ErrorCode.INVALID_ROW,
+        DataValidationUtil::validateAndParseArray,
+        objectMapper.readTree("[]"));
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseArray, new Object());
+    expectError(
+        ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseArray, "foo"); // invalid JSON
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseArray, new Date());
+    expectError(
+        ErrorCode.INVALID_ROW,
+        DataValidationUtil::validateAndParseArray,
+        Collections.singletonList(new Object()));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        DataValidationUtil::validateAndParseArray,
+        Collections.singletonList(Collections.singletonMap("foo", new Object())));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        DataValidationUtil::validateAndParseArray,
+        Collections.singletonMap(new Object(), "foo"));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        DataValidationUtil::validateAndParseArray,
+        Collections.singletonMap("foo", new Object()));
   }
 
   @Test
   public void testValidateAndParseObject() throws Exception {
     String stringObject = "{\"key\":1}";
-    Assert.assertEquals(stringObject, DataValidationUtil.validateAndParseObject(stringObject));
-    JsonNode nodeObject = objectMapper.readTree(stringObject);
-    Assert.assertEquals(stringObject, DataValidationUtil.validateAndParseObject(nodeObject));
+    assertEquals(stringObject, validateAndParseObject(stringObject));
 
     String badObject = "foo";
     try {
-      DataValidationUtil.validateAndParseObject(badObject);
+      validateAndParseObject(badObject);
       Assert.fail("Expected INVALID_ROW error");
     } catch (SFException err) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), err.getVendorCode());
+      assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), err.getVendorCode());
     }
 
     char[] data = new char[20000000];
@@ -427,11 +472,239 @@ public void testValidateAndParseObject() throws Exception {
     mapVal.put("key", stringVal);
     String tooLargeObject = objectMapper.writeValueAsString(mapVal);
     try {
-      DataValidationUtil.validateAndParseObject(tooLargeObject);
+      validateAndParseObject(tooLargeObject);
       Assert.fail("Expected INVALID_ROW error");
     } catch (SFException err) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), err.getVendorCode());
+      assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), err.getVendorCode());
     }
+
+    // Test forbidden values
+    expectError(
+        ErrorCode.INVALID_ROW,
+        DataValidationUtil::validateAndParseObject,
+        objectMapper.readTree("{}"));
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, "[]");
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, "1");
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, 1);
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, 1.5);
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, false);
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, new Object());
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, "foo");
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, new Date());
+    expectError(
+        ErrorCode.INVALID_ROW,
+        DataValidationUtil::validateAndParseObject,
+        Collections.singletonList(new Object()));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        DataValidationUtil::validateAndParseObject,
+        Collections.singletonList(Collections.singletonMap("foo", new Object())));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        DataValidationUtil::validateAndParseObject,
+        Collections.singletonMap(new Object(), "foo"));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        DataValidationUtil::validateAndParseObject,
+        Collections.singletonMap("foo", new Object()));
+  }
+
+  @Test
+  public void testTooLargeVariant() {
+    char[] stringContent = new char[16 * 1024 * 1024 - 16]; // {"a":"11","b":""}
+    Arrays.fill(stringContent, 'c');
+
+    // {"a":"11","b":""}
+    Map<String, Object> m = new HashMap<>();
+    m.put("a", "11");
+    m.put("b", new String(stringContent));
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseVariant, m);
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseArray, m);
+    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, m);
+  }
+
+  @Test
+  public void testTooLargeMultiByteSemiStructuredValues() {
+    // Variant max size is not in characters, but in bytes
+    char[] stringContent = new char[9 * 1024 * 1024]; // 8MB < value < 16MB
+    Arrays.fill(stringContent, 'Č');
+
+    Map<String, Object> m = new HashMap<>();
+    m.put("a", new String(stringContent));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: {a=ČČČČČČČČČČČČČČČČČ.... Value cannot"
+            + " be ingested into Snowflake column VARIANT: Variant too long: length=18874376"
+            + " maxLength=16777152",
+        () -> validateAndParseVariant(m));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: [{\"a\":\"ČČČČČČČČČČČČČ.... Value"
+            + " cannot be ingested into Snowflake column ARRAY: Array too large. length=18874378"
+            + " maxLength=16777152",
+        () -> validateAndParseArray(m));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: {\"a\":\"ČČČČČČČČČČČČČČ.... Value"
+            + " cannot be ingested into Snowflake column OBJECT: Object too large. length=18874376"
+            + " maxLength=16777152",
+        () -> validateAndParseObject(m));
+  }
+
+  @Test
+  public void testValidVariantType() {
+    // Test primitive types
+    Assert.assertTrue(isAllowedSemiStructuredType((byte) 1));
+    Assert.assertTrue(isAllowedSemiStructuredType((short) 1));
+    Assert.assertTrue(isAllowedSemiStructuredType(1));
+    Assert.assertTrue(isAllowedSemiStructuredType(1L));
+    Assert.assertTrue(isAllowedSemiStructuredType(1.25f));
+    Assert.assertTrue(isAllowedSemiStructuredType(1.25d));
+    Assert.assertTrue(isAllowedSemiStructuredType(false));
+    Assert.assertTrue(isAllowedSemiStructuredType('c'));
+
+    // Test boxed primitive types
+    Assert.assertTrue(isAllowedSemiStructuredType(Byte.valueOf((byte) 1)));
+    Assert.assertTrue(isAllowedSemiStructuredType(Short.valueOf((short) 1)));
+    Assert.assertTrue(isAllowedSemiStructuredType(Integer.valueOf(1)));
+    Assert.assertTrue(isAllowedSemiStructuredType(Long.valueOf(1L)));
+    Assert.assertTrue(isAllowedSemiStructuredType(Float.valueOf(1.25f)));
+    Assert.assertTrue(isAllowedSemiStructuredType(Double.valueOf(1.25d)));
+    Assert.assertTrue(isAllowedSemiStructuredType(Boolean.valueOf(false)));
+    Assert.assertTrue(isAllowedSemiStructuredType(Character.valueOf('c')));
+
+    // Test primitive arrays
+    Assert.assertTrue(isAllowedSemiStructuredType(new byte[] {1}));
+    Assert.assertTrue(isAllowedSemiStructuredType(new short[] {1}));
+    Assert.assertTrue(isAllowedSemiStructuredType(new int[] {1}));
+    Assert.assertTrue(isAllowedSemiStructuredType(new long[] {1L}));
+    Assert.assertTrue(isAllowedSemiStructuredType(new float[] {1.25f}));
+    Assert.assertTrue(isAllowedSemiStructuredType(new double[] {1.25d}));
+    Assert.assertTrue(isAllowedSemiStructuredType(new boolean[] {false}));
+    Assert.assertTrue(isAllowedSemiStructuredType(new char[] {'c'}));
+
+    // Test primitive lists
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonList((byte) 1)));
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonList((short) 1)));
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonList(1)));
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonList(1L)));
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonList(1.25f)));
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonList(1.25d)));
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonList(false)));
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonList('c')));
+
+    // Test additional numeric types and their collections
+    Assert.assertTrue(isAllowedSemiStructuredType(new BigInteger("1")));
+    Assert.assertTrue(isAllowedSemiStructuredType(new BigInteger[] {new BigInteger("1")}));
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonList(new BigInteger("1"))));
+    Assert.assertTrue(isAllowedSemiStructuredType(new BigDecimal("1.25")));
+    Assert.assertTrue(isAllowedSemiStructuredType(new BigDecimal[] {new BigDecimal("1.25")}));
+    Assert.assertTrue(
+        isAllowedSemiStructuredType(Collections.singletonList(new BigDecimal("1.25"))));
+
+    // Test strings
+    Assert.assertTrue(isAllowedSemiStructuredType("foo"));
+    Assert.assertTrue(isAllowedSemiStructuredType(new String[] {"foo"}));
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonList("foo")));
+
+    // Test date/time objects and their collections
+    Assert.assertTrue(isAllowedSemiStructuredType(LocalTime.now()));
+    Assert.assertTrue(isAllowedSemiStructuredType(OffsetTime.now()));
+    Assert.assertTrue(isAllowedSemiStructuredType(LocalDate.now()));
+    Assert.assertTrue(isAllowedSemiStructuredType(LocalDateTime.now()));
+    Assert.assertTrue(isAllowedSemiStructuredType(ZonedDateTime.now()));
+    Assert.assertTrue(isAllowedSemiStructuredType(OffsetDateTime.now()));
+    Assert.assertTrue(isAllowedSemiStructuredType(new LocalTime[] {LocalTime.now()}));
+    Assert.assertTrue(isAllowedSemiStructuredType(new OffsetTime[] {OffsetTime.now()}));
+    Assert.assertTrue(isAllowedSemiStructuredType(new LocalDate[] {LocalDate.now()}));
+    Assert.assertTrue(isAllowedSemiStructuredType(new LocalDateTime[] {LocalDateTime.now()}));
+    Assert.assertTrue(isAllowedSemiStructuredType(new ZonedDateTime[] {ZonedDateTime.now()}));
+    Assert.assertTrue(isAllowedSemiStructuredType(new OffsetDateTime[] {OffsetDateTime.now()}));
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonList(LocalTime.now())));
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonList(OffsetTime.now())));
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonList(LocalDate.now())));
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonList(LocalDateTime.now())));
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonList(ZonedDateTime.now())));
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonList(OffsetDateTime.now())));
+
+    // Test mixed collections
+    Assert.assertTrue(
+        isAllowedSemiStructuredType(
+            new Object[] {
+              1,
+              false,
+              new BigInteger("1"),
+              LocalDateTime.now(),
+              new Object[] {new Object[] {new Object[] {LocalDateTime.now(), false}}}
+            }));
+    Assert.assertFalse(
+        isAllowedSemiStructuredType(
+            new Object[] {
+              1,
+              false,
+              new BigInteger("1"),
+              LocalDateTime.now(),
+              new Object[] {new Object[] {new Object[] {new Object(), false}}}
+            }));
+    Assert.assertTrue(
+        isAllowedSemiStructuredType(
+            Arrays.asList(
+                new BigInteger("1"),
+                "foo",
+                false,
+                Arrays.asList(13, Arrays.asList(Arrays.asList(false, 'c'))))));
+    Assert.assertFalse(
+        isAllowedSemiStructuredType(
+            Arrays.asList(
+                new BigInteger("1"),
+                "foo",
+                false,
+                Arrays.asList(13, Arrays.asList(Arrays.asList(new Object(), 'c'))))));
+
+    // Test maps
+    Assert.assertTrue(isAllowedSemiStructuredType(Collections.singletonMap("foo", "bar")));
+    Assert.assertFalse(isAllowedSemiStructuredType(Collections.singletonMap(new Object(), "foo")));
+    Assert.assertFalse(isAllowedSemiStructuredType(Collections.singletonMap("foo", new Object())));
+    Assert.assertTrue(
+        isAllowedSemiStructuredType(
+            Collections.singletonMap(
+                "foo",
+                new Object[] {
+                  1,
+                  false,
+                  new BigInteger("1"),
+                  LocalDateTime.now(),
+                  new Object[] {new Object[] {new Object[] {LocalDateTime.now(), false}}}
+                })));
+    Assert.assertFalse(
+        isAllowedSemiStructuredType(
+            Collections.singletonMap(
+                "foo",
+                new Object[] {
+                  1,
+                  false,
+                  new BigInteger("1"),
+                  LocalDateTime.now(),
+                  new Object[] {new Object[] {new Object[] {new Object(), false}}}
+                })));
+    Assert.assertTrue(
+        isAllowedSemiStructuredType(
+            Collections.singletonMap(
+                "foo",
+                Arrays.asList(
+                    new BigInteger("1"),
+                    "foo",
+                    false,
+                    Arrays.asList(13, Arrays.asList(Arrays.asList(false, 'c')))))));
+    Assert.assertFalse(
+        isAllowedSemiStructuredType(
+            Collections.singletonMap(
+                "foo",
+                Arrays.asList(
+                    new BigInteger("1"),
+                    "foo",
+                    false,
+                    Arrays.asList(13, Arrays.asList(Arrays.asList(new Object(), 'c')))))));
   }
 
   @Test
@@ -471,17 +744,6 @@ public void testValidateAndParseDate() {
         ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, BigDecimal.valueOf(1.25));
   }
 
-  @Test
-  public void testGetStringValue() throws Exception {
-    Assert.assertEquals("123", DataValidationUtil.getStringValue("123"));
-    Assert.assertEquals("123", DataValidationUtil.getStringValue(123));
-    Assert.assertEquals("123", DataValidationUtil.getStringValue(new BigDecimal("123")));
-    Assert.assertEquals("123", DataValidationUtil.getStringValue(new BigInteger("123")));
-    Assert.assertEquals("123.0", DataValidationUtil.getStringValue(123f));
-    Assert.assertEquals("123.0", DataValidationUtil.getStringValue(123d));
-    Assert.assertEquals("123", DataValidationUtil.getStringValue(123l));
-  }
-
   @Test
   public void testValidateAndParseBinary() {
     byte[] maxAllowedArray = new byte[BYTES_8_MB];
@@ -724,5 +986,44 @@ public void testExceptionMessages() {
         "The given row cannot be converted to Arrow format: ghi. Value cannot be ingested into"
             + " Snowflake column BINARY: Not a valid hex string",
         () -> validateAndParseBinary("ghi", Optional.empty()));
+
+    // VARIANT
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
+            + " be ingested into Snowflake column of type VARIANT. Allowed Java types: String,"
+            + " Primitive data types and their arrays, java.time.*, List<T>, Map<String, T>, T[]",
+        () -> validateAndParseVariant(new Object()));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: ][. Value cannot be ingested into"
+            + " Snowflake column VARIANT: Not a valid JSON",
+        () -> validateAndParseVariant("]["));
+
+    // ARRAY
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
+            + " be ingested into Snowflake column of type ARRAY. Allowed Java types: String,"
+            + " Primitive data types and their arrays, java.time.*, List<T>, Map<String, T>, T[]",
+        () -> validateAndParseArray(new Object()));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: ][. Value cannot be ingested into"
+            + " Snowflake column ARRAY: Not a valid JSON",
+        () -> validateAndParseArray("]["));
+
+    // OBJECT
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
+            + " be ingested into Snowflake column of type OBJECT. Allowed Java types: String,"
+            + " Primitive data types and their arrays, java.time.*, List<T>, Map<String, T>, T[]",
+        () -> validateAndParseObject(new Object()));
+    expectErrorCodeAndMessage(
+        ErrorCode.INVALID_ROW,
+        "The given row cannot be converted to Arrow format: }{. Value cannot be ingested into"
+            + " Snowflake column OBJECT: Not a valid JSON",
+        () -> validateAndParseObject("}{"));
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index a5e59d68c..0ea0e5de4 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -284,16 +284,6 @@ <STREAMING_INGEST_WRITE> void assertVariant(
       String expectedValue,
       String expectedType)
       throws Exception {
-    assertVariant(dataType, streamingIngestWriteValue, expectedValue, expectedType, true);
-  }
-
-  <STREAMING_INGEST_WRITE> void assertVariant(
-      String dataType,
-      STREAMING_INGEST_WRITE streamingIngestWriteValue,
-      String expectedValue,
-      String expectedType,
-      boolean compareAsJsons)
-      throws Exception {
 
     String tableName = createTable(dataType);
     String offsetToken = UUID.randomUUID().toString();
@@ -317,9 +307,7 @@ <STREAMING_INGEST_WRITE> void assertVariant(
     }
 
     Assert.assertEquals(1, counter);
-    if (compareAsJsons)
-      Assert.assertEquals(objectMapper.readTree(expectedValue), objectMapper.readTree(value));
-    else Assert.assertEquals(expectedValue, value);
+    Assert.assertEquals(objectMapper.readTree(expectedValue), objectMapper.readTree(value));
     Assert.assertEquals(expectedType, typeof);
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
new file mode 100644
index 000000000..57867427d
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
@@ -0,0 +1,33 @@
+package net.snowflake.ingest.streaming.internal.datatypes;
+
+import java.util.Arrays;
+import org.junit.Test;
+
+public class NullIT extends AbstractDataTypeTest {
+  @Test
+  public void testNullIngestion() throws Exception {
+    for (String type :
+        Arrays.asList(
+            "INT",
+            "NUMBER(1, 1)",
+            "NUMBER(1, 0)",
+            "BOOLEAN",
+            "VARCHAR",
+            "BINARY",
+            "DATE",
+            "TIME",
+            "TIME(0)",
+            "TIMESTAMP_NTZ",
+            "TIMESTAMP_NTZ(0)",
+            "TIMESTAMP_LTZ",
+            "TIMESTAMP_LTZ(0)",
+            "TIMESTAMP_TZ",
+            "TIMESTAMP_TZ(0)",
+            "VARIANT",
+            "ARRAY",
+            "OBJECT")) {
+      // Provider type does not matter as we are expecting null anyway
+      testIngestion(type, null, null, new StringProvider());
+    }
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
new file mode 100644
index 000000000..3c4ddde22
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
@@ -0,0 +1,216 @@
+package net.snowflake.ingest.streaming.internal.datatypes;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.OffsetTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class SemiStructuredIT extends AbstractDataTypeTest {
+
+  // TODO SNOW-664249: There is a few-byte mismatch between the value sent by the user and its
+  // server-side representation. Validation leaves a small buffer for this difference.
+  private static final int MAX_ALLOWED_LENGTH = 16 * 1024 * 1024 - 64;
+
+  @Test
+  public void testVariant() throws Exception {
+    // Test dates
+    assertVariant("VARIANT", LocalTime.of(0, 0, 0, 123), "\"00:00:00.000000123\"", "VARCHAR");
+    assertVariant(
+        "VARIANT",
+        OffsetTime.of(0, 0, 0, 123, ZoneOffset.ofHours(-1)),
+        "\"00:00:00.000000123-01:00\"",
+        "VARCHAR");
+    assertVariant("VARIANT", LocalDate.of(1970, 1, 1), "\"1970-01-01\"", "VARCHAR");
+    assertVariant(
+        "VARIANT", LocalDateTime.of(1970, 1, 1, 0, 0, 0), "\"1970-01-01T00:00\"", "VARCHAR");
+    assertVariant(
+        "VARIANT",
+        Instant.EPOCH.atOffset(ZoneOffset.ofHours(-1)),
+        "\"1969-12-31T23:00-01:00\"",
+        "VARCHAR");
+    assertVariant(
+        "VARIANT",
+        Instant.EPOCH.atZone(ZoneId.of("America/Los_Angeles")),
+        "\"1969-12-31T16:00-08:00\"",
+        "VARCHAR");
+
+    // Test arrays
+    assertVariant("VARIANT", "[]", "[]", "ARRAY");
+    assertVariant("VARIANT", "[1]", "[1]", "ARRAY");
+    assertVariant(
+        "VARIANT", new byte[] {Byte.MIN_VALUE, 0, Byte.MAX_VALUE}, "[-128,0,127]", "ARRAY");
+    assertVariant(
+        "VARIANT", Arrays.asList(Byte.MIN_VALUE, 0, Byte.MAX_VALUE), "[-128,0,127]", "ARRAY");
+    assertVariant(
+        "VARIANT", new short[] {Short.MIN_VALUE, 0, Short.MAX_VALUE}, "[-32768,0,32767]", "ARRAY");
+    assertVariant(
+        "VARIANT", Arrays.asList(Short.MIN_VALUE, 0, Short.MAX_VALUE), "[-32768,0,32767]", "ARRAY");
+    assertVariant(
+        "VARIANT",
+        new int[] {Integer.MIN_VALUE, 0, Integer.MAX_VALUE},
+        "[-2147483648,0,2147483647]",
+        "ARRAY");
+    assertVariant(
+        "VARIANT",
+        Arrays.asList(Integer.MIN_VALUE, 0, Integer.MAX_VALUE),
+        "[-2147483648,0,2147483647]",
+        "ARRAY");
+    assertVariant(
+        "VARIANT",
+        new long[] {Long.MIN_VALUE, 0L, Long.MAX_VALUE},
+        "[-9223372036854775808,0,9223372036854775807]",
+        "ARRAY");
+    assertVariant(
+        "VARIANT",
+        Arrays.asList(Long.MIN_VALUE, 0L, Long.MAX_VALUE),
+        "[-9223372036854775808,0,9223372036854775807]",
+        "ARRAY");
+    assertVariant(
+        "VARIANT",
+        new Object[] {
+          Byte.MIN_VALUE,
+          Short.MAX_VALUE,
+          0,
+          Long.MAX_VALUE,
+          new Object[2],
+          Collections.emptyMap(),
+          null
+        },
+        "[-128,32767,0,9223372036854775807,[null,null],{}, null]",
+        "ARRAY");
+    assertVariant(
+        "VARIANT",
+        Arrays.asList(
+            Byte.MIN_VALUE,
+            Short.MAX_VALUE,
+            0,
+            Long.MAX_VALUE,
+            new Object[2],
+            Collections.emptyMap(),
+            null),
+        "[-128,32767,0,9223372036854775807,[null,null],{}, null]",
+        "ARRAY");
+    assertVariant("VARIANT", new BigInteger[] {BigInteger.TEN}, "[10]", "ARRAY");
+    assertVariant("VARIANT", Collections.singletonList(BigInteger.TEN), "[10]", "ARRAY");
+    assertVariant("VARIANT", new BigDecimal[] {new BigDecimal("10.5")}, "[10.5]", "ARRAY");
+    assertVariant("VARIANT", Collections.singletonList(new BigDecimal("10.5")), "[10.5]", "ARRAY");
+
+    // Test booleans
+    assertVariant("VARIANT", "false", "false", "BOOLEAN");
+    assertVariant("VARIANT", false, "false", "BOOLEAN");
+    assertVariant("VARIANT", true, "true", "BOOLEAN");
+
+    // Test numbers
+    assertVariant("VARIANT", 34, "34", "INTEGER");
+    assertVariant("VARIANT", "34", "34", "INTEGER");
+
+    assertVariant("VARIANT", Byte.MAX_VALUE, String.valueOf(Byte.MAX_VALUE), "INTEGER");
+    assertVariant("VARIANT", Short.MAX_VALUE, String.valueOf(Short.MAX_VALUE), "INTEGER");
+    assertVariant("VARIANT", Integer.MAX_VALUE, String.valueOf(Integer.MAX_VALUE), "INTEGER");
+    assertVariant("VARIANT", Long.MAX_VALUE, String.valueOf(Long.MAX_VALUE), "INTEGER");
+    assertVariant("VARIANT", Long.MIN_VALUE, String.valueOf(Long.MIN_VALUE), "INTEGER");
+    assertVariant("VARIANT", BigInteger.TEN, BigInteger.TEN.toString(), "INTEGER");
+    assertVariant(
+        "VARIANT", new BigDecimal("10.54"), new BigDecimal("10.54").toString(), "DECIMAL");
+    assertVariant("VARIANT", "-1000.25", "-1000.25", "DECIMAL");
+    assertVariant("VARIANT", -1000.25f, "-1000.25", "DECIMAL");
+    assertVariant("VARIANT", -1000.25, "-1000.25", "DECIMAL");
+
+    // Test objects
+    assertVariant("VARIANT", "{}", "{}", "OBJECT");
+    String testObject =
+        "{\"a\": \"foo\", \"b\":15, \"c\": false, \"d\": [1, 2, 3], \"e\": {\"q\": false}, \"f\":"
+            + " null}";
+    assertVariant("VARIANT", testObject, testObject, "OBJECT");
+    assertVariant(
+        "VARIANT",
+        Collections.singletonMap(
+            "A",
+            Collections.singletonMap("B", Collections.singletonMap("C", new String[] {"foo"}))),
+        "{\"A\": {\"B\": {\"C\": [\"foo\"]}}}",
+        "OBJECT");
+
+    // Test strings
+    assertVariant("VARIANT", "\"foo\"", "\"foo\"", "VARCHAR");
+    assertVariant("VARIANT", '1', "\"1\"", "VARCHAR");
+    assertVariant("VARIANT", 'd', "\"d\"", "VARCHAR");
+    assertVariant(
+        "VARIANT", "\"ž, š, č, ř, c, j, ď, ť, ň\"", "\"ž, š, č, ř, c, j, ď, ť, ň\"", "VARCHAR");
+
+    // Date/time strings are ingested as varchars
+    assertVariant("VARIANT", "\"13:05:33.299094\"", "\"13:05:33.299094\"", "VARCHAR");
+    assertVariant("VARIANT", "\"13:05:55.684003Z\"", "\"13:05:55.684003Z\"", "VARCHAR");
+    assertVariant("VARIANT", "\"2022-09-14\"", "\"2022-09-14\"", "VARCHAR");
+    assertVariant(
+        "VARIANT", "\"2022-09-14T13:04:53.578667\"", "\"2022-09-14T13:04:53.578667\"", "VARCHAR");
+    assertVariant(
+        "VARIANT",
+        "\"2022-09-14T06:16:09.797704-07:00[America/Los_Angeles]\"",
+        "\"2022-09-14T06:16:09.797704-07:00[America/Los_Angeles]\"",
+        "VARCHAR");
+
+    // Test JSON null
+    assertVariant("VARIANT", "null", "null", "NULL_VALUE");
+  }
+
+  @Test
+  public void testMaxVariantAndObject() throws Exception {
+    String maxObject = createLargeVariantObject(MAX_ALLOWED_LENGTH);
+    assertVariant("VARIANT", maxObject, maxObject, "OBJECT");
+    assertVariant("OBJECT", maxObject, maxObject, "OBJECT");
+  }
+
+  @Test
+  public void testMaxArray() throws Exception {
+    String maxArray = "[" + createLargeVariantObject(MAX_ALLOWED_LENGTH - 2) + "]";
+    assertVariant("ARRAY", maxArray, maxArray, "ARRAY");
+  }
+
+  @Test
+  public void testObject() throws Exception {
+    assertVariant("OBJECT", "{}", "{}", "OBJECT");
+    assertVariant("OBJECT", Collections.emptyMap(), "{}", "OBJECT");
+    assertVariant("OBJECT", Collections.singletonMap("1", 2), "{\"1\": 2}", "OBJECT");
+    assertVariant(
+        "OBJECT", Collections.singletonMap("1", new byte[] {1, 2}), "{\"1\": [1,2]}", "OBJECT");
+  }
+
+  @Test
+  public void testArray() throws Exception {
+    assertVariant("ARRAY", "[]", "[]", "ARRAY");
+    assertVariant("ARRAY", Collections.emptyList(), "[]", "ARRAY");
+    assertVariant("ARRAY", new Object[0], "[]", "ARRAY");
+    assertVariant("ARRAY", new int[0], "[]", "ARRAY");
+    assertVariant("ARRAY", new double[0], "[]", "ARRAY");
+
+    assertVariant("ARRAY", 1, "[1]", "ARRAY");
+    assertVariant("ARRAY", "null", "[null]", "ARRAY");
+    assertVariant("ARRAY", "{}", "[{}]", "ARRAY");
+    assertVariant("ARRAY", Collections.emptyMap(), "[{}]", "ARRAY");
+    assertVariant("ARRAY", Collections.singletonMap("1", "2"), "[{\"1\": \"2\"}]", "ARRAY");
+  }
+
+  private String createLargeVariantObject(int size) throws JsonProcessingException {
+    char[] stringContent = new char[size - 17]; // {"a":"11","b":""}
+    Arrays.fill(stringContent, 'c');
+    Map<String, Object> m = new HashMap<>();
+    m.put("a", "11");
+    m.put("b", new String(stringContent));
+    String s = new ObjectMapper().writeValueAsString(m);
+    Assert.assertEquals(s.length(), size);
+    return s;
+  }
+}

From 5bd8f10462e2511a4a6d04366bba57bf641bae44 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 10 Nov 2022 12:45:36 -0800
Subject: [PATCH 070/356] 1.0.2-beta.6 release

---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index e60d2d989..b81ea6e37 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
     <!-- Arifact name and version information -->
     <groupId>net.snowflake</groupId>
     <artifactId>snowflake-ingest-sdk</artifactId>
-    <version>1.0.2-beta.5</version>
+    <version>1.0.2-beta.6</version>
     <packaging>jar</packaging>
     <name>Snowflake Ingest SDK</name>
     <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 07085d582..2b438a8dd 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -127,7 +127,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.0.2-beta.5";
+  public static final String DEFAULT_VERSION = "1.0.2-beta.6";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From bc082a97d3016db36a3155acea5cbaf6573e1200 Mon Sep 17 00:00:00 2001
From: Harsh Pathak <harsh.pathak@snowflake.com>
Date: Wed, 28 Sep 2022 19:50:30 -0700
Subject: [PATCH 071/356] Delete old semgrep workflow

---
 .github/workflows/semgrep.yml | 14 --------------
 1 file changed, 14 deletions(-)
 delete mode 100644 .github/workflows/semgrep.yml

diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml
deleted file mode 100644
index 713a25109..000000000
--- a/.github/workflows/semgrep.yml
+++ /dev/null
@@ -1,14 +0,0 @@
-name: Semgrep
-on:
-    pull_request:
-        branches:
-            - master
-jobs:
-  semgrep:
-    runs-on: ubuntu-latest
-    name: Semgrep
-    steps:
-    - uses: actions/checkout@v1
-    - name: Semgrep
-      id: semgrep
-      uses: returntocorp/semgrep-action@v1

From 6cf58e50b89d5926e942bfb4db0d62a576d9acb0 Mon Sep 17 00:00:00 2001
From: Harsh Pathak <harsh.pathak@snowflake.com>
Date: Wed, 28 Sep 2022 19:50:31 -0700
Subject: [PATCH 072/356] Delete old semgrep workflow

---
 .github/workflows/semgrep_internal.yml | 18 ------------------
 1 file changed, 18 deletions(-)
 delete mode 100644 .github/workflows/semgrep_internal.yml

diff --git a/.github/workflows/semgrep_internal.yml b/.github/workflows/semgrep_internal.yml
deleted file mode 100644
index e3f0d8835..000000000
--- a/.github/workflows/semgrep_internal.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-name: Semgrep
-
-on:
-  pull_request: {}
-
-jobs:
-  semgrep:
-    name: Scan
-    runs-on: ubuntu-latest
-    if: (github.actor != 'dependabot[bot]')
-    steps:
-      - uses: actions/checkout@v2
-      - uses: returntocorp/semgrep-action@v1
-        with:
-          auditOn: push
-          publishUrl: https://semgrep.snowflake.com
-          publishDeployment: 1
-          publishToken: ${{ secrets.SEMGREP_APP_TOKEN }}

From 65f0d33d5f73b6f0cf39f6d49c6a656e338020fb Mon Sep 17 00:00:00 2001
From: Harsh Pathak <harsh.pathak@snowflake.com>
Date: Wed, 28 Sep 2022 19:50:32 -0700
Subject: [PATCH 073/356] Add new semgrep workflow file

---
 .github/workflows/semgrep.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 .github/workflows/semgrep.yml

diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml
new file mode 100644
index 000000000..d2a49777e
--- /dev/null
+++ b/.github/workflows/semgrep.yml
@@ -0,0 +1,16 @@
+---
+name: Run semgrep checks
+
+on:
+  pull_request:
+    branches: [master]
+
+permissions:
+  contents: read
+
+jobs:
+  run-semgrep-reusable-workflow:
+    uses: snowflakedb/reusable-workflows/.github/workflows/semgrep-v2.yml@main
+    secrets:
+      token: ${{ secrets.SEMGREP_APP_TOKEN }}
+

From ec5423a77d140dacc9cedb6dcbe3af1184f0b6e3 Mon Sep 17 00:00:00 2001
From: Harsh Pathak <harsh.pathak@snowflake.com>
Date: Wed, 28 Sep 2022 19:50:33 -0700
Subject: [PATCH 074/356] Delete local semgrep rules file

---
 .semgrep.yml | 425 ---------------------------------------------------
 1 file changed, 425 deletions(-)
 delete mode 100644 .semgrep.yml

diff --git a/.semgrep.yml b/.semgrep.yml
deleted file mode 100644
index cd0425af0..000000000
--- a/.semgrep.yml
+++ /dev/null
@@ -1,425 +0,0 @@
-rules:
-- id: java.lang.security.audit.bad-hexa-conversion.bad-hexa-conversion
-  metadata:
-    cwe: 'CWE-704: Incorrect Type Conversion or Cast'
-    owasp: 'A3: Sensitive Data Exposure'
-    source-rule-url: https://find-sec-bugs.github.io/bugs.htm#BAD_HEXA_CONVERSION
-  message: |
-    This mistake weakens the hash value computed since it introduces more collisions.
-    In this situation, the method Integer.toHexString() should be replaced with String.format().
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#bad-hexa-conversion
-  severity: WARNING
-  languages: [java]
-  pattern: |-
-    $X $METHOD(...) {
-      ...
-      MessageDigest $MD = ...;
-      ...
-      $MD.digest(...);
-      ...
-      Integer.toHexString(...);
-    }
-- id: java.lang.security.audit.cbc-padding-oracle.cbc-padding-oracle
-  message: |
-    Using CBC with PKCS5Padding is susceptible to padding orcale attacks. A malicious actor
-    could discern the difference between plaintext with valid or invalid padding. Further,
-    CBC mode does not include any integrity checks (https://find-sec-bugs.github.io/bugs.htm#CIPHER_INTEGRITY).
-    Use 'AES/GCM/NoPadding' instead.
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#cbc-padding-oracle
-  metadata:
-    cwe: 'CWE-696: Incorrect Behavior Order'
-    owasp: 'A3: Sensitive Data Exposure'
-    source-rule-url: https://find-sec-bugs.github.io/bugs.htm#PADDING_ORACLE
-    references:
-    - https://capec.mitre.org/data/definitions/463.html
-  severity: WARNING
-  patterns:
-  - pattern: $CIPHER.getInstance("=~/.*\/CBC\/PKCS5Padding/");
-  fix: $CIPHER.getInstance("AES/GCM/NoPadding");
-  languages:
-  - java
-- id: java.lang.security.audit.command-injection-formatted-runtime-call.command-injection-formatted-runtime-call
-  patterns:
-  - pattern-either:
-    - pattern: $RUNTIME.exec($X + $Y);
-    - pattern: $RUNTIME.exec(String.format(...));
-    - pattern: $RUNTIME.loadLibrary($X + $Y);
-    - pattern: $RUNTIME.loadLibrary(String.format(...));
-  message: |
-    A formatted or concatenated string was detected as input to a java.lang.Runtime call.
-    This is dangerous if a variable is controlled by user input and could result in a
-    command injection. Ensure your variables are not controlled by users or sufficiently sanitized.
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#command-injection-formatted-runtime-call
-  metadata:
-    cwe: "CWE-78: Improper Neutralization of Special Elements used in an OS Command\
-      \ ('OS Command Injection')"
-    owasp: 'A1: Injection'
-    source-rule-url: https://find-sec-bugs.github.io/bugs.htm#COMMAND_INJECTION.
-  severity: WARNING
-  languages:
-  - java
-- id: java.lang.security.audit.script-engine-injection.script-engine-injection
-  message: |
-    Potential code injection when using Script Engine.
-    Ensure that untrusted data is not being passed to this function or otherwise ensure that proper sandboxing is being performed.
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#script-engine-injection
-  metadata:
-    cwe: "CWE-94: Improper Control of Generation of Code ('Code Injection')"
-    owasp: 'A1: Injection'
-    source-rule-url: https://find-sec-bugs.github.io/bugs.htm#SCRIPT_ENGINE_INJECTION
-  severity: WARNING
-  languages: [java]
-  patterns:
-  - pattern-either:
-    - pattern-inside: |
-        class $CLASS {
-          ...
-          ScriptEngine $SE;
-          ...
-        }
-    - pattern-inside: |
-        class $CLASS {
-          ...
-          ScriptEngine $SE = ...;
-          ...
-        }
-    - pattern-inside: |
-        $X $METHOD(...) {
-          ...
-          ScriptEngine $SE = ...;
-          ...
-        }
-  - pattern: |
-      $X $METHOD(...) {
-        ...
-        $SE.eval(...);
-        ...
-      }
-  - pattern-not: |
-      $X $METHOD(...) {
-        ...
-        $SE.eval("...");
-        ...
-      }
-  - pattern-not: |
-      $X $METHOD(...) {
-        ...
-        String $S = "...";
-        ...
-        $SE.eval($S);
-        ...
-      }
-- id: java.lang.security.audit.weak-ssl-context.weak-ssl-context
-  metadata:
-    cwe: 'CWE-326: Inadequate Encryption Strength'
-    owasp: 'A3: Sensitive Data Exposure'
-    source_rule_url: https://find-sec-bugs.github.io/bugs.htm#SSL_CONTEXT
-    references:
-    - https://tools.ietf.org/html/rfc7568
-    - https://tools.ietf.org/id/draft-ietf-tls-oldversions-deprecate-02.html
-  message: |
-    An insecure SSL context was detected. TLS versions 1.0, 1.1, and all SSL versions
-    are considered weak encryption and are deprecated.
-    Use SSLContext.getInstance("TLSv1.2") or SSLContext.getInstance("TLSv1.3") for the best security.
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#weak-ssl-context
-  severity: WARNING
-  languages: [java]
-  patterns:
-  - pattern-not: SSLContext.getInstance("TLS1.3")
-  - pattern-not: SSLContext.getInstance("TLS1.2")
-  - pattern: SSLContext.getInstance("...")
-- id: java.lang.security.audit.xml-decoder.xml-decoder
-  message: |
-    XMLDecoder should not be used to parse untrusted data. Deserializing user input can lead to arbitrary code execution.
-    Ensure that only trusted data is being parsed using XMLDecoder.
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#xml-decoder
-  metadata:
-    cwe: 'CWE-611: Improper Restriction of XML External Entity Reference'
-    owasp: 'A4: XML External Entities (XXE)'
-    source-rule-url: https://find-sec-bugs.github.io/bugs.htm#XML_DECODER
-  severity: WARNING
-  languages: [java]
-  patterns:
-  - pattern: |
-      $X $METHOD(...) {
-        ...
-        new XMLDecoder(...);
-        ...
-      }
-  - pattern-not: |
-      $X $METHOD(...) {
-        ...
-        new XMLDecoder("...");
-        ...
-      }
-  - pattern-not: |-
-      $X $METHOD(...) {
-        ...
-        String $STR = "...";
-        ...
-        new XMLDecoder($STR);
-        ...
-      }
-- id: java.lang.security.audit.xssrequestwrapper-is-insecure.xssrequestwrapper-is-insecure
-  metadata:
-    owasp: 'A7: Cross-Site Scripting (XSS)'
-    cwe: "CWE-79: Improper Neutralization of Input During Web Page Generation ('Cross-site\
-      \ Scripting')"
-    source-rule-url: https://find-sec-bugs.github.io/bugs.htm#XSS_REQUEST_WRAPPER
-  message: |
-    It looks like you're using an implementation of XSSRequestWrapper from dzone.
-    (https://www.javacodegeeks.com/2012/07/anti-cross-site-scripting-xss-filter.html)
-    The XSS filtering in this code is not secure and can be bypassed by malicious actors.
-    It is recommended to use a stack that automatically escapes in your view or templates
-    instead of filtering yourself.
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#xssrequestwrapper-is-insecure
-  severity: WARNING
-  patterns:
-  - pattern-either:
-    - pattern: |
-        class XSSRequestWrapper extends HttpServletRequestWrapper {
-          ...
-        }
-    - pattern: |-
-        $P = $X.compile("</script>", $X.CASE_INSENSITIVE);
-        $V = $P.matcher(...).replaceAll("");
-  languages:
-  - java
-- id: java.lang.security.audit.cookie-missing-httponly.cookie-missing-httponly
-  metadata:
-    cwe: "CWE-1004: Sensitive Cookie Without 'HttpOnly' Flag"
-    owasp: 'A3: Sensitive Data Exposure'
-    source-rule-url: https://find-sec-bugs.github.io/bugs.htm#HTTPONLY_COOKIE
-  message: |
-    A cookie was detected without setting the 'HttpOnly' flag. The 'HttpOnly' flag
-    for cookies instructs the browser to forbid client-side scripts from reading the
-    cookie. Set the 'HttpOnly' flag by calling 'cookie.setHttpOnly(true);'
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#cookie-missing-httponly
-  severity: WARNING
-  languages: [java]
-  patterns:
-  - pattern-not-inside: $COOKIE.setValue(""); ...
-  - pattern-either:
-    - pattern: $COOKIE.setHttpOnly(false);
-    - patterns:
-      - pattern-not-inside: $COOKIE.setHttpOnly(...); ...
-      - pattern: $RESPONSE.addCookie($COOKIE);
-- id: java.lang.security.audit.cookie-missing-samesite.cookie-missing-samesite
-  metadata:
-    cwe: 'CWE-352: Cross-Site Request Forgery (CSRF)'
-    owasp: 'A6: Security Misconfiguration'
-    references:
-    - https://stackoverflow.com/questions/42717210/samesite-cookie-in-java-application
-  message: |
-    Detected cookie without the SameSite attribute.
-    Set the SameSite attributed on the cookie.
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#cookie-missing-samesite
-  severity: WARNING
-  languages: [java]
-  patterns:
-  - pattern-not-inside: |
-      $RETURNTYPE $METHOD(..., HttpServletResponse $RESP, ...) {
-        ...
-        $RESP.setHeader("Set-Cookie", "=~/.*SameSite=.*/");
-        ...
-      }
-  - pattern-either:
-    - pattern: $RESP.addCookie(...);
-    - pattern: $RESP.setHeader("Set-Cookie", ...);
-- id: java.lang.security.audit.cookie-missing-secure-flag.cookie-missing-secure-flag
-  metadata:
-    cwe: "CWE-614: Sensitive Cookie in HTTPS Session Without 'Secure' Attribute"
-    owasp: 'A3: Sensitive Data Exposure'
-    source-rule-url: https://find-sec-bugs.github.io/bugs.htm#INSECURE_COOKIE
-  message: |
-    A cookie was detected without setting the 'secure' flag. The 'secure' flag
-    for cookies prevents the client from transmitting the cookie over insecure
-    channels such as HTTP. Set the 'secure' flag by calling '$COOKIE.setSecure(true);'
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#cookie-missing-secure-flag
-  severity: WARNING
-  languages: [java]
-  patterns:
-  - pattern-not-inside: $COOKIE.setValue(""); ...
-  - pattern-either:
-    - pattern: (Cookie $COOKIE).setSecure(false);
-    - patterns:
-      - pattern-not-inside: (Cookie $COOKIE).setSecure(...); ...
-      - pattern: $RESPONSE.addCookie($COOKIE);
-- id: java.lang.security.audit.crypto.no-static-initialization-vector.no-static-initialization-vector
-  message: |
-    Initialization Vectors (IVs) for block ciphers should be randomly generated
-    each time they are used. Using a static IV means the same plaintext
-    encrypts to the same ciphertext every time, weakening the strength
-    of the encryption.
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#no-static-initialization-vector
-  metadata:
-    cwe: 'CWE-329: Not Using a Random IV with CBC Mode'
-    owasp: 'A3: Sensitive Data Exposure'
-    source-rule-url: https://find-sec-bugs.github.io/bugs.htm#STATIC_IV
-    references:
-    - https://cwe.mitre.org/data/definitions/329.html
-  severity: WARNING
-  languages: [java]
-  patterns:
-  - pattern-either:
-    - pattern: |
-        byte[] $IV = {
-            ...
-        };
-        ...
-        new IvParameterSpec($IV, ...);
-    - pattern: |
-        class $CLASS {
-            byte[] $IV = {
-                ...
-            };
-            ...
-            $METHOD(...) {
-                ...
-                new IvParameterSpec($IV, ...);
-                ...
-            }
-        }
-- id: java.lang.security.audit.crypto.weak-hash.use-of-sha1
-  message: |
-    Use of weak cryptographic primitive SHA1
-    Use SHA256 at minimum for signature verification and use PBKDF2 for hashing passwords.
-    For more informations, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#use-of-sha1
-  languages: [java]
-  severity: WARNING
-  metadata:
-    owasp: 'A9: Using Components with Known Vulnerabilities'
-    cwe: 'CWE-327: Use of a Broken or Risky Cryptographic Algorithm'
-    source-rule-url: https://find-sec-bugs.github.io/bugs.htm#WEAK_MESSAGE_DIGEST_SHA1
-  pattern-either:
-  - pattern: |
-      MessageDigest $VAR = $MD.getInstance("SHA1");
-  - pattern: |
-      $DU.getSha1Digest().digest(...)
-- id: java.lang.security.audit.crypto.weak-hash.use-of-md5
-  message: |
-    Use of weak cryptographic primitive MD5
-    Use SHA256 at minimum for signature verification and use PBKDF2 for hashing passwords.
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#use-of-md5
-  languages: [java]
-  severity: WARNING
-  metadata:
-    owasp: 'A9: Using Components with Known Vulnerabilities'
-    cwe: 'CWE-327: Use of a Broken or Risky Cryptographic Algorithm'
-    source-rule-url: https://find-sec-bugs.github.io/bugs.htm#WEAK_MESSAGE_DIGEST_MD5
-  pattern-either:
-  - pattern: |
-      MessageDigest $VAR = $MD.getInstance("MD5");
-  - pattern: |
-      $DU.getMd5Digest().digest(...)
-- id: java.lang.security.audit.crypto.no-null-cipher.no-null-cipher
-  pattern: new NullCipher(...);
-  metadata:
-    cwe: 'CWE-327: Use of a Broken or Risky Cryptographic Algorithm'
-    owasp: 'A3: Sensitive Data Exposure'
-    source-rule-url: https://find-sec-bugs.github.io/bugs.htm#NULL_CIPHER
-  message: |
-    NullCipher was detected. This will not encrypt anything;
-    the cipher text will be the same as the plain text. Use
-    a valid, secure cipher: Cipher.getInstance("AES/GCM/NoPadding").
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#no-null-cipher
-  severity: WARNING
-  languages:
-  - java
-- id: java.lang.security.audit.crypto.ssl.avoid-implementing-custom-digests.avoid-implementing-custom-digests
-  metadata:
-    cwe: 'CWE-327: Use of a Broken or Risky Cryptographic Algorithm'
-    owasp: 'A3: Sensitive Data Exposure'
-    source-rule-url: https://find-sec-bugs.github.io/bugs.htm#CUSTOM_MESSAGE_DIGEST
-    references:
-    - https://cheatsheetseries.owasp.org/cheatsheets/Cryptographic_Storage_Cheat_Sheet.html#custom-algorithms
-  message: |
-    Cryptographic algorithms are notoriously difficult to get right. By implementing
-    a custom message digest, you risk introducing security issues into your program.
-    Use one of the many sound message digests already available to you:
-    MessageDigest sha256Digest = MessageDigest.getInstance("SHA256");
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#avoid-implementing-custom-digests
-  severity: WARNING
-  languages: [java]
-  pattern: |-
-    class $CLASS extends MessageDigest {
-      ...
-    }
-- id: java.lang.security.audit.crypto.ssl.defaulthttpclient-is-deprecated.defaulthttpclient-is-deprecated
-  metadata:
-    cwe: 'CWE-326: Inadequate Encryption Strength'
-    owasp: 'A3: Sensitive Data Exposure'
-    source-rule-url: https://find-sec-bugs.github.io/bugs.htm#DEFAULT_HTTP_CLIENT
-  message: |
-    DefaultHttpClient is deprecated. Further, it does not support connections
-    using TLS1.2, which makes using DefaultHttpClient a security hazard.
-    Use SystemDefaultHttpClient instead, which supports TLS1.2.
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#defaulthttpclient-is-deprecated
-  severity: WARNING
-  languages: [java]
-  pattern: new DefaultHttpClient(...);
-- id: java.lang.security.audit.crypto.ssl.insecure-hostname-verifier.insecure-hostname-verifier
-  message: |
-    Insecure HostnameVerifier implementation detected. This will accept
-    any SSL certificate with any hostname, which creates the possibility
-    for man-in-the-middle attacks.
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#insecure-hostname-verifier
-  metadata:
-    cwe: 'CWE-295: Improper Certificate Validation'
-    owasp: 'A6: Security Misconfiguration'
-    source-rule-url: https://find-sec-bugs.github.io/bugs.htm#WEAK_HOSTNAME_VERIFIER
-  severity: WARNING
-  languages: [java]
-  patterns:
-  - pattern-either:
-    - pattern: |
-        class $CLASS implements HostnameVerifier {
-          ...
-          public boolean verify(...) { return true; }
-        }
-    - pattern: |-
-        new HostnameVerifier(...){
-          public boolean verify(...) {
-            return true;
-          }
-        }
-- id: java.lang.security.audit.crypto.ssl.insecure-trust-manager.insecure-trust-manager
-  metadata:
-    cwe: 'CWE-295: Improper Certificate Validation'
-    owasp: 'A3: Sensitive Data Exposure'
-    source-rule-url: https://find-sec-bugs.github.io/bugs.htm#WEAK_TRUST_MANAGER
-    references:
-    - https://stackoverflow.com/questions/2642777/trusting-all-certificates-using-httpclient-over-https
-  message: |
-    Detected empty trust manager implementations. This is dangerous because it accepts any
-    certificate, enabling man-in-the-middle attacks. Consider using a KeyStore
-    and TrustManagerFactory isntead.
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#insecure-trust-manager
-  severity: WARNING
-  languages: [java]
-  patterns:
-  - pattern-inside: |
-      class $CLASS implements X509TrustManager {
-        ...
-      }
-  - pattern-not: public void checkClientTrusted(...) { $SOMETHING; }
-  - pattern-not: public void checkServerTrusted(...) { $SOMETHING; }
-  - pattern-either:
-    - pattern: public void checkClientTrusted(...) {}
-    - pattern: public void checkServerTrusted(...) {}
-    - pattern: public X509Certificate[] getAcceptedIssuers(...) { return null; }
-- id: java.lang.correctness.eqeq.eqeq
-  patterns:
-  - pattern-not-inside: assert $X;
-  - pattern-not-inside: |
-      assert $X : $Y;
-  - pattern-either:
-    - pattern: $X == $X
-    - pattern: $X != $X
-  - pattern-not: 1 == 1
-  message: |
-    '`$X == $X` or `$X != $X` is a useless comparison unless the value compared
-    is a float or double. To test if `$X` is not-a-number, use `Double.isNaN($X)`.'
-    For more information, please refer to https://snowflakecomputing.atlassian.net/wiki/spaces/CLO/pages/1127713128/Semgrep+Finding+Remediation#eqeq
-  languages: [java]
-  severity: ERROR

From c39157a1f937c4d655a8d9ee37bd50957cab4aa0 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Fri, 30 Sep 2022 12:54:20 -0700
Subject: [PATCH 075/356] V1.0.2-beta.5 release (#235)

This release contains a few bug fixes and improvements for Snowpipe Streaming:

[Improvement] Improve and fix a few data type validation logic
[Improvement] Add throttling logic on direct memory based on uploading queue size
[Improvement] Add basic logic to support Parquet file format, this work is still in progress and we're still using Arrow
[Improvement] Improve exception handling logic
---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index b81ea6e37..e60d2d989 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
     <!-- Arifact name and version information -->
     <groupId>net.snowflake</groupId>
     <artifactId>snowflake-ingest-sdk</artifactId>
-    <version>1.0.2-beta.6</version>
+    <version>1.0.2-beta.5</version>
     <packaging>jar</packaging>
     <name>Snowflake Ingest SDK</name>
     <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 2b438a8dd..07085d582 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -127,7 +127,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.0.2-beta.6";
+  public static final String DEFAULT_VERSION = "1.0.2-beta.5";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From d4e1a9195461cf522c8f02b532e0e3847d3ab39a Mon Sep 17 00:00:00 2001
From: sfc-gh-snyk-sca-sa
 <95290361+sfc-gh-snyk-sca-sa@users.noreply.github.com>
Date: Tue, 4 Oct 2022 01:26:38 +0530
Subject: [PATCH 076/356] SNOW-470176: [Snyk] Security upgrade
 com.fasterxml.jackson.core:jackson-databind from 2.13.2.1 to 2.13.4 (#237)

fix: pom.xml to reduce vulnerabilities

The following vulnerabilities are fixed with an upgrade:
- https://snyk.io/vuln/SNYK-JAVA-COMFASTERXMLJACKSONCORE-3038424

Co-authored-by: snyk-bot <snyk-bot@snyk.io>
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index e60d2d989..2ac071626 100644
--- a/pom.xml
+++ b/pom.xml
@@ -221,7 +221,7 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.13.2.1</version>
+            <version>2.13.4</version>
         </dependency>
 
 

From efee2c2f90bef57e1b1dcd6061a3b365a64adb60 Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Tue, 11 Oct 2022 14:50:02 +0200
Subject: [PATCH 077/356] @snow SNOW-633432 Parquet file generator (#200)

SNOW-633432 Parquet file generator
Co-authored-by: Gloria Doci, Andrey Zagrebin
---
 pom.xml                                       |  60 ++
 scripts/check_content.sh                      |  18 +-
 .../streaming/internal/ArrowRowBuffer.java    |  10 +-
 .../streaming/internal/BlobMetadata.java      |  17 +-
 .../internal/BlobMetadataWithBdecVersion.java |  25 +
 .../streaming/internal/ChannelCache.java      |   3 +-
 .../internal/DataValidationUtil.java          |  11 +
 .../streaming/internal/FlushService.java      |   7 +-
 .../ingest/streaming/internal/Flusher.java    |   3 +-
 .../streaming/internal/ParquetChunkData.java  |  25 +
 .../streaming/internal/ParquetFlusher.java    | 348 +++++++++++
 .../streaming/internal/ParquetRowBuffer.java  | 222 +++++++
 .../internal/ParquetTypeGenerator.java        | 292 +++++++++
 .../internal/ParquetValueParser.java          | 323 ++++++++++
 .../streaming/internal/RegisterService.java   |   3 +-
 .../ingest/streaming/internal/RowBuffer.java  |   3 +-
 ...owflakeStreamingIngestChannelInternal.java |  20 +-
 ...nowflakeStreamingIngestClientInternal.java |   5 +-
 .../net/snowflake/ingest/utils/Constants.java |   9 +-
 .../java/net/snowflake/ingest/TestUtils.java  |  12 +-
 .../streaming/internal/ArrowBufferTest.java   | 242 ++++----
 .../internal/ColumnMetadataBuilder.java       | 164 +++++
 .../streaming/internal/FlushServiceTest.java  |  50 +-
 .../internal/ParquetTypeGeneratorTest.java    | 480 +++++++++++++++
 .../internal/ParquetValueParserTest.java      | 570 ++++++++++++++++++
 .../streaming/internal/RowBufferTest.java     |   7 +-
 .../streaming/internal/StreamingIngestIT.java | 289 ++++++---
 .../datatypes/AbstractDataTypeTest.java       |  32 +-
 .../internal/datatypes/BinaryIT.java          |   5 +
 .../internal/datatypes/DateTimeIT.java        |   5 +
 .../internal/datatypes/LogicalTypesIT.java    |   6 +
 .../streaming/internal/datatypes/NullIT.java  |   6 +
 .../internal/datatypes/NumericTypesIT.java    |   6 +
 .../internal/datatypes/SemiStructuredIT.java  |   5 +
 .../internal/datatypes/StringsIT.java         |   6 +
 35 files changed, 3027 insertions(+), 262 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java

diff --git a/pom.xml b/pom.xml
index 2ac071626..e243039bd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -43,6 +43,7 @@
         <jacoco.skip.instrument>true</jacoco.skip.instrument>
         <jacoco.version>0.8.5</jacoco.version>
         <arrow.version>9.0.0</arrow.version>
+        <hadoop.version>2.10.1</hadoop.version>
     </properties>
 
     <build>
@@ -302,6 +303,65 @@
             <version>${arrow.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-hadoop</artifactId>
+            <version>1.11.2</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>javax.annotation</groupId>
+                    <artifactId>javax.annotation-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>${hadoop.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.github.spotbugs</groupId>
+                    <artifactId>spotbugs-annotations</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-json</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-server</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>javax.servlet</groupId>
+                    <artifactId>servlet-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>javax.servlet.jsp</groupId>
+                    <artifactId>jsp-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.codehaus.jackson</groupId>
+                    <artifactId>jackson-mapper-asl</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.mortbay.jetty</groupId>
+                    <artifactId>jetty</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
         <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-core -->
         <dependency>
             <groupId>io.dropwizard.metrics</groupId>
diff --git a/scripts/check_content.sh b/scripts/check_content.sh
index 9e5b1c404..d2594d583 100755
--- a/scripts/check_content.sh
+++ b/scripts/check_content.sh
@@ -5,7 +5,23 @@
 set -o pipefail
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-if jar tvf $DIR/../target/snowflake-ingest-sdk.jar  | awk '{print $8}' | grep -v -E "^(net|com)/snowflake" | grep -v -E "(com|net)/\$" | grep -v -E "^META-INF" | grep -v -E "^mozilla" | grep -v mime.types | grep -v project.properties | grep -v -E "javax" | grep -v -E "^org/" | grep -v -E "^com/google" | grep -v -E "^com/sun" | grep -v "log4j.properties" | grep -v "git.properties" | grep -v "io/" | grep -v "codegen/" | grep -v "com/codahale/" | grep -v "com/ibm/" | grep -v "LICENSE" | grep -v "aix/" | grep -v "darwin/" | grep -v "win/" | grep -v "freebsd/" | grep -v "linux/" | grep -v "com/github/"; then
+if jar tvf $DIR/../target/snowflake-ingest-sdk.jar  | awk '{print $8}' | grep -v -E "^(net|com)/snowflake" \
+    | grep -v -E "(com|net)/\$" | grep -v -E "^META-INF" | grep -v -E "^mozilla" | grep -v mime.types \
+    | grep -v project.properties | grep -v -E "javax" | grep -v -E "^org/" | grep -v -E "^com/google" \
+    | grep -v -E "^com/sun" | grep -v "log4j.properties" | grep -v "git.properties" | grep -v "io/" \
+    | grep -v "codegen/" | grep -v "com/codahale/" | grep -v "com/ibm/" | grep -v "LICENSE" | grep -v "aix/" \
+    | grep -v "darwin/" | grep -v "win/" | grep -v "freebsd/" | grep -v "linux/" | grep -v "com/github/" \
+    | grep -v -E "shaded/" | grep -v "webapps/"  | grep -v "microsoft/" | grep -v "com/ctc/" \
+    | grep -v "edu/" | grep -v "com/jcraft/" | grep -v "contribs/" | grep -v "com/zaxxer/"  | grep -v "com/squareup/" \
+    | grep -v "com/thoughtworks/" | grep -v "com/jamesmurty/"  | grep -v "net/iharder/" \
+    | grep -v -E "^core-default.xml" \
+    | grep -v "yarn-version-info.properties" | grep -v "yarn-version-info.properties" \
+    | grep -v "common-version-info.properties" | grep -v "LocalizedFormats_fr.properties" \
+    | grep -v "org.apache.hadoop.application-classloader.properties" | grep -v "assets/" | grep -v "ehcache-core.xsd" \
+    | grep -v "ehcache-107ext.xsd" | grep -v "parquet.thrift" | grep -v "mapred-default.xml" \
+    | grep -v "yarn-default.xml" | grep -v ".keep"  | grep -v "NOTICE" | grep -v "digesterRules.xml" \
+    | grep -v "properties.dtd" | grep -v "PropertyList-1.0.dtd"  \
+    ; then
   echo "[ERROR] Ingest SDK jar includes class not under the snowflake namespace"
   exit 1
 fi
\ No newline at end of file
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 38b4583a9..2d7d17a92 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -471,15 +471,7 @@ private float convertRowToArrow(
             // vector.setSafe requires the BigDecimal input scale explicitly match its scale
             inputAsBigDecimal = inputAsBigDecimal.setScale(columnScale, RoundingMode.HALF_UP);
 
-            if (inputAsBigDecimal.abs().compareTo(BigDecimal.TEN.pow(columnPrecision - columnScale))
-                >= 0) {
-              throw new SFException(
-                  ErrorCode.INVALID_ROW,
-                  inputAsBigDecimal,
-                  String.format(
-                      "Number out of representable exclusive range of (-1e%s..1e%s)",
-                      columnPrecision - columnScale, columnPrecision - columnScale));
-            }
+            DataValidationUtil.checkValueInRange(inputAsBigDecimal, columnScale, columnPrecision);
 
             if (columnScale != 0 || physicalType == ColumnPhysicalType.SB16) {
               ((DecimalVector) vector).setSafe(curRowIndex, inputAsBigDecimal);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index 2818a392a..f90ba5055 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -49,9 +49,16 @@ List<ChunkMetadata> getChunks() {
     return this.chunks;
   }
 
-  // TODO: send the bdec_version once server side supports this in production
-  //  @JsonProperty("bdec_version")
-  //  byte getVersionByte() {
-  //    return bdecVersion.toByte();
-  //  }
+  /**
+   * Create {@link BlobMetadata} in case of {@link Constants.BdecVersion#ONE} and {@link
+   * BlobMetadataWithBdecVersion} otherwise to send BDEC version to server side.
+   */
+  static BlobMetadata createBlobMetadata(
+      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks) {
+    // TODO SNOW-659721: Unify BlobMetadata with BlobMetadataWithBdecVersion once server side
+    // BdecVersion in production
+    return bdecVersion == Constants.BdecVersion.ONE
+        ? new BlobMetadata(path, md5, bdecVersion, chunks)
+        : new BlobMetadataWithBdecVersion(path, md5, bdecVersion, chunks);
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
new file mode 100644
index 000000000..7f9c602e3
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+import net.snowflake.ingest.utils.Constants;
+
+/**
+ * Same as {@link BlobMetadata} but with {@link Constants.BdecVersion} to send it to server side if
+ * the version is greater than {@link Constants.BdecVersion#ONE}.
+ */
+class BlobMetadataWithBdecVersion extends BlobMetadata {
+  BlobMetadataWithBdecVersion(
+      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks) {
+    super(path, md5, bdecVersion, chunks);
+  }
+
+  @JsonProperty("bdec_version")
+  byte getVersionByte() {
+    return getVersion().toByte();
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
index 35ca580bc..ef366b120 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
@@ -14,7 +14,8 @@
  * during flush. The key is a fully qualified table name and the value is a set of channels that
  * belongs to this table
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 class ChannelCache<T> {
   // Cache to hold all the valid channels, the key for the outer map is FullyQualifiedTableName and
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 881a4c2f3..de7185009 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -674,6 +674,17 @@ static int validateAndParseBoolean(Object input) {
         input.getClass(), "BOOLEAN", new String[] {"boolean", "Number", "String"});
   }
 
+  static void checkValueInRange(BigDecimal bigDecimalValue, int scale, int precision) {
+    if (bigDecimalValue.abs().compareTo(BigDecimal.TEN.pow(precision - scale)) >= 0) {
+      throw new SFException(
+          ErrorCode.INVALID_ROW,
+          bigDecimalValue,
+          String.format(
+              "Number out of representable exclusive range of (-1e%s..1e%s)",
+              precision - scale, precision - scale));
+    }
+  }
+
   static Set<String> allowedBooleanStringsLowerCased =
       Sets.newHashSet("1", "0", "yes", "no", "y", "n", "t", "f", "true", "false", "on", "off");
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index df5522469..0ef790c6f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -58,7 +58,8 @@
  * <li>upload the blob to stage
  * <li>register the blob to the targeted Snowflake table
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 class FlushService<T> {
 
@@ -467,7 +468,6 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
         // We need to maintain IV as a block counter for the whole file, even interleaved,
         // to align with decryption on the Snowflake query path.
         // TODO: address alignment for the header SNOW-557866
-        // TODO: encryption is not yet supported by server side for Parquet
         long iv = curDataSize / Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES;
         byte[] encryptedCompressedChunkData =
             Cryptor.encrypt(
@@ -559,7 +559,8 @@ BlobMetadata upload(String filePath, byte[] blob, List<ChunkMetadata> metadata)
         blob.length,
         System.currentTimeMillis() - startTime);
 
-    return new BlobMetadata(filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata);
+    return BlobMetadata.createBlobMetadata(
+        filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
index 4ac7acf8c..37ab4afc4 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
@@ -13,7 +13,8 @@
  * Interface to convert {@link ChannelData} buffered in {@link RowBuffer} to the underlying format
  * implementation for faster processing.
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 public interface Flusher<T> {
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
new file mode 100644
index 000000000..0c7b12ff2
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.util.List;
+import java.util.Map;
+
+/** Parquet data holder to buffer rows. */
+public class ParquetChunkData {
+  final List<List<Object>> rows;
+  final Map<String, String> metadata;
+
+  /**
+   * Construct parquet data chunk.
+   *
+   * @param rows chunk row set
+   * @param metadata chunk metadata
+   */
+  public ParquetChunkData(List<List<Object>> rows, Map<String, String> metadata) {
+    this.rows = rows;
+    this.metadata = metadata;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
new file mode 100644
index 000000000..3d9827b6b
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
@@ -0,0 +1,348 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import static net.snowflake.ingest.utils.Constants.MAX_CHUNK_SIZE_IN_BYTES;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.ParquetProperties;
+import org.apache.parquet.hadoop.ParquetFileWriter;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.api.WriteSupport;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.io.DelegatingPositionOutputStream;
+import org.apache.parquet.io.OutputFile;
+import org.apache.parquet.io.ParquetEncodingException;
+import org.apache.parquet.io.PositionOutputStream;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.api.RecordConsumer;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+
+/**
+ * Converts {@link ChannelData} buffered in {@link RowBuffer} to the Parquet format for faster
+ * processing.
+ */
+public class ParquetFlusher implements Flusher<ParquetChunkData> {
+  private static final Logging logger = new Logging(ParquetFlusher.class);
+  private final MessageType schema;
+
+  /** Construct parquet flusher from its schema. */
+  public ParquetFlusher(MessageType schema) {
+    this.schema = schema;
+  }
+
+  @Override
+  public SerializationResult serialize(
+      List<ChannelData<ParquetChunkData>> channelsDataPerTable,
+      ByteArrayOutputStream chunkData,
+      String filePath)
+      throws IOException {
+    List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
+    long rowCount = 0L;
+    SnowflakeStreamingIngestChannelInternal<ParquetChunkData> firstChannel = null;
+    Map<String, RowBufferStats> columnEpStatsMapCombined = null;
+    List<List<Object>> rows = null;
+
+    for (ChannelData<ParquetChunkData> data : channelsDataPerTable) {
+      // Create channel metadata
+      ChannelMetadata channelMetadata =
+          ChannelMetadata.builder()
+              .setOwningChannel(data.getChannel())
+              .setRowSequencer(data.getRowSequencer())
+              .setOffsetToken(data.getOffsetToken())
+              .build();
+      // Add channel metadata to the metadata list
+      channelsMetadataList.add(channelMetadata);
+
+      logger.logDebug(
+          "Parquet Flusher: Start building channel={}, rowCount={}, bufferSize={} in blob={}",
+          data.getChannel().getFullyQualifiedName(),
+          data.getRowCount(),
+          data.getBufferSize(),
+          filePath);
+
+      if (rows == null) {
+        columnEpStatsMapCombined = data.getColumnEps();
+        rows = new ArrayList<>();
+        firstChannel = data.getChannel();
+      } else {
+        // This method assumes that channelsDataPerTable is grouped by table. We double check
+        // here and throw an error if the assumption is violated
+        if (!data.getChannel()
+            .getFullyQualifiedTableName()
+            .equals(firstChannel.getFullyQualifiedTableName())) {
+          throw new SFException(ErrorCode.INVALID_DATA_IN_CHUNK);
+        }
+
+        columnEpStatsMapCombined =
+            ChannelData.getCombinedColumnStatsMap(columnEpStatsMapCombined, data.getColumnEps());
+      }
+      rows.addAll(data.getVectors().rows);
+
+      rowCount += data.getRowCount();
+
+      logger.logDebug(
+          "Parquet Flusher: Finish building channel={}, rowCount={}, bufferSize={} in blob={}",
+          data.getChannel().getFullyQualifiedName(),
+          data.getRowCount(),
+          data.getBufferSize(),
+          filePath);
+    }
+
+    Map<String, String> metadata = channelsDataPerTable.get(0).getVectors().metadata;
+
+    flushToParquetBdecChunk(chunkData, rows, metadata, channelsMetadataList);
+    return new SerializationResult(channelsMetadataList, columnEpStatsMapCombined, rowCount);
+  }
+
+  /**
+   * Flushes a parquet row chunk to the given BDEC output stream.
+   *
+   * @param bdecOutput BDEC output stream
+   * @param chunkRows chunk rows
+   * @param metadata chunk metadata
+   * @param channelsMetadataList metadata of the channels the chunk rows belong to
+   * @throws IOException thrown from Parquet library in case of writing problems
+   */
+  private void flushToParquetBdecChunk(
+      ByteArrayOutputStream bdecOutput,
+      List<List<Object>> chunkRows,
+      Map<String, String> metadata,
+      List<ChannelMetadata> channelsMetadataList)
+      throws IOException {
+    try {
+      ParquetWriter<List<Object>> writer =
+          new BdecParquetWriterBuilder(bdecOutput, schema, metadata, channelsMetadataList)
+              // PARQUET_2_0 uses Encoding.DELTA_BYTE_ARRAY for byte arrays (e.g. SF sb16)
+              // server side does not support it TODO: SNOW-657238
+              .withWriterVersion(ParquetProperties.WriterVersion.PARQUET_1_0)
+
+              // the dictionary encoding (Encoding.*_DICTIONARY) is not supported by server side
+              // scanner yet
+              .withDictionaryEncoding(false)
+
+              // Historically server side scanner supports only the case when the row number in all
+              // pages is the same.
+              // The quick fix is to effectively disable the page size/row limit
+              // to always have one page per chunk until server side is generalised.
+              .withPageSize((int) Constants.MAX_CHUNK_SIZE_IN_BYTES * 2)
+              .withPageRowCountLimit(chunkRows.size() + 1)
+              .enableValidation()
+              .withCompressionCodec(CompressionCodecName.GZIP)
+              .withWriteMode(ParquetFileWriter.Mode.CREATE)
+              .build();
+
+      // We can use lower level column writers and custom ValuesWriterFactory that uses plain byte
+      // array encoding used by PARQUET_1_0 and supported by server side
+      // TODO: SNOW-672143
+      for (List<Object> row : chunkRows) {
+        writer.write(row);
+      }
+      writer.close();
+    } catch (Throwable t) {
+      logger.logError("Parquet Flusher: failed to write", t);
+      throw t;
+    }
+  }
+
+  /**
+   * A parquet specific write builder.
+   *
+   * <p>This class is implemented as parquet library API requires, mostly to provide {@link
+   * BdecWriteSupport} implementation.
+   */
+  private static class BdecParquetWriterBuilder
+      extends ParquetWriter.Builder<List<Object>, BdecParquetWriterBuilder> {
+    private final MessageType schema;
+    private final Map<String, String> extraMetaData;
+    private final List<ChannelMetadata> channelsMetadataList;
+
+    protected BdecParquetWriterBuilder(
+        ByteArrayOutputStream stream,
+        MessageType schema,
+        Map<String, String> extraMetaData,
+        List<ChannelMetadata> channelsMetadataList) {
+      super(new ByteArrayOutputFile(stream));
+      this.schema = schema;
+      this.extraMetaData = extraMetaData;
+      this.channelsMetadataList = channelsMetadataList;
+    }
+
+    @Override
+    protected BdecParquetWriterBuilder self() {
+      return this;
+    }
+
+    @Override
+    protected WriteSupport<List<Object>> getWriteSupport(Configuration conf) {
+      return new BdecWriteSupport(schema, extraMetaData, channelsMetadataList);
+    }
+  }
+
+  /**
+   * A parquet specific file output implementation.
+   *
+   * <p>This class is implemented as parquet library API requires, mostly to create our {@link
+   * ByteArrayDelegatingPositionOutputStream} implementation.
+   */
+  private static class ByteArrayOutputFile implements OutputFile {
+    private final ByteArrayOutputStream stream;
+
+    private ByteArrayOutputFile(ByteArrayOutputStream stream) {
+      this.stream = stream;
+    }
+
+    @Override
+    public PositionOutputStream create(long blockSizeHint) throws IOException {
+      stream.reset();
+      return new ByteArrayDelegatingPositionOutputStream(stream);
+    }
+
+    @Override
+    public PositionOutputStream createOrOverwrite(long blockSizeHint) throws IOException {
+      return create(blockSizeHint);
+    }
+
+    @Override
+    public boolean supportsBlockSize() {
+      return false;
+    }
+
+    @Override
+    public long defaultBlockSize() {
+      return (int) MAX_CHUNK_SIZE_IN_BYTES;
+    }
+  }
+
+  /**
+   * A parquet specific output stream implementation.
+   *
+   * <p>This class is implemented as parquet library API requires, mostly to wrap our BDEC output
+   * {@link ByteArrayOutputStream}.
+   */
+  private static class ByteArrayDelegatingPositionOutputStream
+      extends DelegatingPositionOutputStream {
+    private final ByteArrayOutputStream stream;
+
+    public ByteArrayDelegatingPositionOutputStream(ByteArrayOutputStream stream) {
+      super(stream);
+      this.stream = stream;
+    }
+
+    @Override
+    public long getPos() {
+      return stream.size();
+    }
+  }
+
+  /**
+   * A parquet specific write support implementation.
+   *
+   * <p>This class is implemented as parquet library API requires, mostly to serialize user column
+   * values depending on type into Parquet {@link RecordConsumer} in {@link
+   * BdecWriteSupport#write(List)}.
+   */
+  private static class BdecWriteSupport extends WriteSupport<List<Object>> {
+    MessageType schema;
+    RecordConsumer recordConsumer;
+    Map<String, String> extraMetadata;
+    List<ChannelMetadata> channelsMetadataList;
+
+    // TODO SNOW-672156: support specifying encodings and compression
+    BdecWriteSupport(
+        MessageType schema,
+        Map<String, String> extraMetadata,
+        List<ChannelMetadata> channelsMetadataList) {
+      this.schema = schema;
+      this.extraMetadata = extraMetadata;
+      this.channelsMetadataList = channelsMetadataList;
+    }
+
+    @Override
+    public WriteContext init(Configuration config) {
+      return new WriteContext(schema, extraMetadata);
+    }
+
+    @Override
+    public void prepareForWrite(RecordConsumer recordConsumer) {
+      this.recordConsumer = recordConsumer;
+    }
+
+    @Override
+    public void write(List<Object> values) {
+      List<ColumnDescriptor> cols = schema.getColumns();
+      if (values.size() != cols.size()) {
+        List<String> channelNames =
+            this.channelsMetadataList.stream()
+                .map(ChannelMetadata::getChannelName)
+                .collect(Collectors.toList());
+        throw new ParquetEncodingException(
+            "Invalid input data in channels "
+                + channelNames
+                + ". Expecting "
+                + cols.size()
+                + " columns. Input had "
+                + values.size()
+                + " columns ("
+                + cols
+                + ") : "
+                + values);
+      }
+
+      recordConsumer.startMessage();
+      for (int i = 0; i < cols.size(); ++i) {
+        Object val = values.get(i);
+        // val.length() == 0 indicates a NULL value.
+        if (val != null) {
+          String fieldName = cols.get(i).getPath()[0];
+          recordConsumer.startField(fieldName, i);
+          PrimitiveType.PrimitiveTypeName typeName =
+              cols.get(i).getPrimitiveType().getPrimitiveTypeName();
+          switch (typeName) {
+            case BOOLEAN:
+              recordConsumer.addBoolean((boolean) val);
+              break;
+            case FLOAT:
+              recordConsumer.addFloat((float) val);
+              break;
+            case DOUBLE:
+              recordConsumer.addDouble((double) val);
+              break;
+            case INT32:
+              recordConsumer.addInteger((int) val);
+              break;
+            case INT64:
+              recordConsumer.addLong((long) val);
+              break;
+            case BINARY:
+              recordConsumer.addBinary(Binary.fromString((String) val));
+              break;
+            case FIXED_LEN_BYTE_ARRAY:
+              Binary binary = Binary.fromConstantByteArray((byte[]) val);
+              recordConsumer.addBinary(binary);
+              break;
+            default:
+              throw new ParquetEncodingException(
+                  "Unsupported column type: " + cols.get(i).getPrimitiveType());
+          }
+          recordConsumer.endField(fieldName, i);
+        }
+      }
+      recordConsumer.endMessage();
+    }
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
new file mode 100644
index 000000000..de4b3eadc
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -0,0 +1,222 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+
+/**
+ * The buffer in the Streaming Ingest channel that holds the un-flushed rows, these rows will be
+ * converted to Parquet format for faster processing
+ */
+public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
+  private static final Logging logger = new Logging(ParquetRowBuffer.class);
+
+  private static final String PARQUET_MESSAGE_TYPE_NAME = "bdec";
+
+  private final Map<String, Pair<ColumnMetadata, Integer>> fieldIndex;
+  private final Map<String, String> metadata;
+  private final List<List<Object>> data;
+  private final List<List<Object>> tempData;
+
+  private MessageType schema;
+
+  /**
+   * Construct a ParquetRowBuffer object
+   *
+   * @param channel client channel
+   */
+  ParquetRowBuffer(SnowflakeStreamingIngestChannelInternal<ParquetChunkData> channel) {
+    super(channel);
+    fieldIndex = new HashMap<>();
+    metadata = new HashMap<>();
+    data = new ArrayList<>();
+    tempData = new ArrayList<>();
+  }
+
+  @Override
+  public void setupSchema(List<ColumnMetadata> columns) {
+    fieldIndex.clear();
+    metadata.clear();
+    List<Type> parquetTypes = new ArrayList<>();
+    // Snowflake column id that corresponds to the order in 'columns' received from server
+    // id is required to pack column metadata for the server scanner, e.g. decimal scale and
+    // precision
+    int id = 1;
+    for (ColumnMetadata column : columns) {
+      ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+          ParquetTypeGenerator.generateColumnParquetTypeInfo(column, id);
+      parquetTypes.add(typeInfo.getParquetType());
+      this.metadata.putAll(typeInfo.getMetadata());
+      fieldIndex.put(column.getName(), new Pair<>(column, parquetTypes.size() - 1));
+      if (!column.getNullable()) {
+        addNonNullableFieldName(column.getName());
+      }
+      this.statsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
+
+      if (this.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.ABORT) {
+        this.tempStatsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
+      }
+
+      id++;
+    }
+    schema = new MessageType(PARQUET_MESSAGE_TYPE_NAME, parquetTypes);
+  }
+
+  @Override
+  boolean hasColumn(String name) {
+    return fieldIndex.containsKey(name);
+  }
+
+  @Override
+  float addRow(
+      Map<String, Object> row,
+      int curRowIndex,
+      Map<String, RowBufferStats> statsMap,
+      Set<String> formattedInputColumnNames) {
+    return addRow(row, data, statsMap, formattedInputColumnNames);
+  }
+
+  @Override
+  float addTempRow(
+      Map<String, Object> row,
+      int curRowIndex,
+      Map<String, RowBufferStats> statsMap,
+      Set<String> formattedInputColumnNames) {
+    return addRow(row, tempData, statsMap, formattedInputColumnNames);
+  }
+
+  /**
+   * Adds a row to the parquet buffer.
+   *
+   * @param row row to add
+   * @param out internal buffer to add to
+   * @param statsMap column stats map
+   * @param inputColumnNames list of input column names after formatting
+   * @return row size
+   */
+  private float addRow(
+      Map<String, Object> row,
+      List<List<Object>> out,
+      Map<String, RowBufferStats> statsMap,
+      Set<String> inputColumnNames) {
+    Object[] indexedRow = new Object[fieldIndex.size()];
+    float size = 0F;
+    for (Map.Entry<String, Object> entry : row.entrySet()) {
+      String key = entry.getKey();
+      Object value = entry.getValue();
+      String columnName = formatColumnName(key);
+      int colIndex = fieldIndex.get(columnName).getSecond();
+      RowBufferStats stats = statsMap.get(columnName);
+      ColumnMetadata column = fieldIndex.get(columnName).getFirst();
+      ColumnDescriptor columnDescriptor = schema.getColumns().get(colIndex);
+      PrimitiveType.PrimitiveTypeName typeName =
+          columnDescriptor.getPrimitiveType().getPrimitiveTypeName();
+      ParquetValueParser.ParquetBufferValue valueWithSize =
+          ParquetValueParser.parseColumnValueToParquet(value, column, typeName, stats);
+      indexedRow[colIndex] = valueWithSize.getValue();
+      size += valueWithSize.getSize();
+    }
+    out.add(Arrays.asList(indexedRow));
+
+    for (String columnName : Sets.difference(this.fieldIndex.keySet(), inputColumnNames)) {
+      statsMap.get(columnName).incCurrentNullCount();
+    }
+    return size;
+  }
+
+  @Override
+  void moveTempRowsToActualBuffer(int tempRowCount) {
+    data.addAll(tempData);
+  }
+
+  @Override
+  void clearTempRows() {
+    tempData.clear();
+  }
+
+  @Override
+  boolean hasColumns() {
+    return !fieldIndex.isEmpty();
+  }
+
+  @Override
+  Optional<ParquetChunkData> getSnapshot() {
+    List<List<Object>> oldData = new ArrayList<>();
+    data.forEach(r -> oldData.add(new ArrayList<>(r)));
+    return oldData.isEmpty()
+        ? Optional.empty()
+        : Optional.of(new ParquetChunkData(oldData, metadata));
+  }
+
+  /** Used only for testing. */
+  @Override
+  Object getVectorValueAt(String column, int index) {
+    int colIndex = fieldIndex.get(column).getSecond();
+    Object value = data.get(index).get(colIndex);
+    ColumnMetadata columnMetadata = fieldIndex.get(column).getFirst();
+    String physicalTypeStr = columnMetadata.getPhysicalType();
+    ColumnPhysicalType physicalType = ColumnPhysicalType.valueOf(physicalTypeStr);
+    String logicalTypeStr = columnMetadata.getLogicalType();
+    ColumnLogicalType logicalType = ColumnLogicalType.valueOf(logicalTypeStr);
+    if (logicalType == ColumnLogicalType.FIXED) {
+      if (physicalType == ColumnPhysicalType.SB1) {
+        value = ((Integer) value).byteValue();
+      }
+      if (physicalType == ColumnPhysicalType.SB2) {
+        value = ((Integer) value).shortValue();
+      }
+      if (physicalType == ColumnPhysicalType.SB16) {
+        value = new BigDecimal(new BigInteger((byte[]) value), columnMetadata.getScale());
+      }
+    }
+    if (logicalType == ColumnLogicalType.BINARY && value != null) {
+      value = ((String) value).getBytes(StandardCharsets.UTF_8);
+    }
+    return value;
+  }
+
+  @Override
+  int getTempRowCount() {
+    return tempData.size();
+  }
+
+  @Override
+  void reset() {
+    super.reset();
+    data.clear();
+  }
+
+  @Override
+  public void close(String name) {
+    this.fieldIndex.clear();
+    logger.logInfo(
+        "Trying to close parquet buffer for channel={} from function={}",
+        this.owningChannel.getName(),
+        name);
+  }
+
+  @Override
+  public Flusher<ParquetChunkData> createFlusher(Constants.BdecVersion bdecVerion) {
+    return new ParquetFlusher(schema);
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
new file mode 100644
index 000000000..149ce766c
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
@@ -0,0 +1,292 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+import org.apache.parquet.schema.Types;
+
+/** Generates the Parquet types for the Snowflake's column types */
+public class ParquetTypeGenerator {
+
+  /**
+   * Util class that contains Parquet type and other metadata for that type needed by the Snowflake
+   * server side scanner
+   */
+  static class ParquetTypeInfo {
+    private Type parquetType;
+    private Map<String, String> metadata;
+
+    public Type getParquetType() {
+      return this.parquetType;
+    }
+
+    public Map<String, String> getMetadata() {
+      return this.metadata;
+    }
+
+    public void setParquetType(Type parquetType) {
+      this.parquetType = parquetType;
+    }
+
+    public void setMetadata(Map<String, String> metadata) {
+      this.metadata = metadata;
+    }
+  }
+
+  private static final Set<AbstractRowBuffer.ColumnPhysicalType> TIME_SUPPORTED_PHYSICAL_TYPES =
+      new HashSet<>(
+          Arrays.asList(
+              AbstractRowBuffer.ColumnPhysicalType.SB4, AbstractRowBuffer.ColumnPhysicalType.SB8));
+  private static final Set<AbstractRowBuffer.ColumnPhysicalType>
+      TIMESTAMP_SUPPORTED_PHYSICAL_TYPES =
+          new HashSet<>(
+              Arrays.asList(
+                  AbstractRowBuffer.ColumnPhysicalType.SB8,
+                  AbstractRowBuffer.ColumnPhysicalType.SB16));
+
+  /**
+   * Generate the column parquet type and metadata from the column metadata received from server
+   * side.
+   *
+   * @param column column metadata as received from server side
+   * @param id column id
+   * @return column parquet type
+   */
+  static ParquetTypeInfo generateColumnParquetTypeInfo(ColumnMetadata column, int id) {
+    ParquetTypeInfo res = new ParquetTypeInfo();
+    Type parquetType;
+    Map<String, String> metadata = new HashMap<>();
+    String name = column.getName();
+
+    AbstractRowBuffer.ColumnPhysicalType physicalType;
+    AbstractRowBuffer.ColumnLogicalType logicalType;
+    try {
+      physicalType = AbstractRowBuffer.ColumnPhysicalType.valueOf(column.getPhysicalType());
+      logicalType = AbstractRowBuffer.ColumnLogicalType.valueOf(column.getLogicalType());
+    } catch (IllegalArgumentException e) {
+      throw new SFException(
+          ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
+    }
+
+    metadata.put(Integer.toString(id), logicalType.getOrdinal() + "," + physicalType.getOrdinal());
+
+    // Parquet Type.Repetition in general supports repeated values for the same row column, like a
+    // list of values.
+    // This generator uses only either 0 or 1 value for nullable data type (OPTIONAL: 0 or none
+    // value if it is null)
+    // or exactly 1 value for non-nullable data type (REQUIRED)
+    Type.Repetition repetition =
+        column.getNullable() ? Type.Repetition.OPTIONAL : Type.Repetition.REQUIRED;
+
+    // Handle differently depends on the column logical and physical types
+    switch (logicalType) {
+      case FIXED:
+        parquetType = getFixedColumnParquetType(column, id, physicalType, repetition);
+        break;
+      case ARRAY:
+      case OBJECT:
+      case VARIANT:
+        // mark the column metadata as being an object json for the server side scanner
+        metadata.put(id + ":obj_enc", "1");
+        // parquetType is same as the next one
+      case ANY:
+      case CHAR:
+      case TEXT:
+      case BINARY:
+        parquetType =
+            Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition)
+                .as(LogicalTypeAnnotation.stringType())
+                .id(id)
+                .named(name);
+        break;
+      case TIMESTAMP_LTZ:
+      case TIMESTAMP_NTZ:
+      case TIMESTAMP_TZ:
+        parquetType =
+            getTimeColumnParquetType(
+                column.getScale(),
+                physicalType,
+                logicalType,
+                TIMESTAMP_SUPPORTED_PHYSICAL_TYPES,
+                repetition,
+                id,
+                name);
+        break;
+      case DATE:
+        parquetType =
+            Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition)
+                .as(LogicalTypeAnnotation.dateType())
+                .id(id)
+                .named(name);
+        break;
+      case TIME:
+        parquetType =
+            getTimeColumnParquetType(
+                column.getScale(),
+                physicalType,
+                logicalType,
+                TIME_SUPPORTED_PHYSICAL_TYPES,
+                repetition,
+                id,
+                name);
+        break;
+      case BOOLEAN:
+        parquetType =
+            Types.primitive(PrimitiveType.PrimitiveTypeName.BOOLEAN, repetition).id(id).named(name);
+        break;
+      case REAL:
+        parquetType =
+            Types.primitive(PrimitiveType.PrimitiveTypeName.DOUBLE, repetition).id(id).named(name);
+        break;
+      default:
+        throw new SFException(
+            ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
+    }
+    res.setParquetType(parquetType);
+    res.setMetadata(metadata);
+    return res;
+  }
+
+  /**
+   * Get the parquet type for column of Snowflake FIXED logical type.
+   *
+   * @param column column metadata
+   * @param id column id in Snowflake table schema
+   * @param physicalType Snowflake physical type of column
+   * @param repetition parquet repetition type of column
+   * @return column parquet type
+   */
+  private static Type getFixedColumnParquetType(
+      ColumnMetadata column,
+      int id,
+      AbstractRowBuffer.ColumnPhysicalType physicalType,
+      Type.Repetition repetition) {
+    String name = column.getName();
+    // the LogicalTypeAnnotation.DecimalLogicalTypeAnnotation is used by server side scanner
+    // to discover data type scale and precision
+    LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalLogicalTypeAnnotation =
+        column.getScale() != null && column.getPrecision() != null
+            ? LogicalTypeAnnotation.DecimalLogicalTypeAnnotation.decimalType(
+                column.getScale(), column.getPrecision())
+            : null;
+    Type parquetType;
+    if ((column.getScale() != null && column.getScale() != 0)
+        || physicalType == AbstractRowBuffer.ColumnPhysicalType.SB16) {
+      parquetType =
+          Types.primitive(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, repetition)
+              .length(16)
+              .as(decimalLogicalTypeAnnotation)
+              .id(id)
+              .named(name);
+    } else {
+      switch (physicalType) {
+        case SB1:
+        case SB2:
+        case SB4:
+          parquetType =
+              Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition)
+                  .as(decimalLogicalTypeAnnotation)
+                  .id(id)
+                  .length(4)
+                  .named(name);
+          break;
+        case SB8:
+          parquetType =
+              Types.primitive(PrimitiveType.PrimitiveTypeName.INT64, repetition)
+                  .as(decimalLogicalTypeAnnotation)
+                  .id(id)
+                  .length(8)
+                  .named(name);
+          break;
+        default:
+          throw new SFException(
+              ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
+      }
+    }
+    return parquetType;
+  }
+
+  /**
+   * Get the parquet type for column of a Snowflake time logical type.
+   *
+   * @param scale column scale
+   * @param physicalType Snowflake physical type of column
+   * @param logicalType Snowflake logical type of column
+   * @param supportedPhysicalTypes supported Snowflake physical types for the given column
+   * @param repetition parquet repetition type of column
+   * @param id column id in Snowflake table schema
+   * @param name column name
+   * @return column parquet type
+   */
+  private static Type getTimeColumnParquetType(
+      Integer scale,
+      AbstractRowBuffer.ColumnPhysicalType physicalType,
+      AbstractRowBuffer.ColumnLogicalType logicalType,
+      Set<AbstractRowBuffer.ColumnPhysicalType> supportedPhysicalTypes,
+      Type.Repetition repetition,
+      int id,
+      String name) {
+    if (scale == null || scale > 9 || scale < 0 || !supportedPhysicalTypes.contains(physicalType)) {
+      throw new SFException(
+          ErrorCode.UNKNOWN_DATA_TYPE,
+          "Data type: " + logicalType + ", " + physicalType + ", scale: " + scale);
+    }
+
+    PrimitiveType.PrimitiveTypeName type = getTimePrimitiveType(physicalType);
+    LogicalTypeAnnotation typeAnnotation;
+    int length;
+    switch (physicalType) {
+      case SB4:
+        typeAnnotation = LogicalTypeAnnotation.decimalType(scale, 9);
+        length = 4;
+        break;
+      case SB8:
+        typeAnnotation = LogicalTypeAnnotation.decimalType(scale, 18);
+        length = 8;
+        break;
+      case SB16:
+        typeAnnotation = LogicalTypeAnnotation.decimalType(scale, 38);
+        length = 16;
+        break;
+      default:
+        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
+    }
+    return Types.primitive(type, repetition).as(typeAnnotation).length(length).id(id).named(name);
+  }
+
+  /**
+   * Get the parquet primitive type name for column of a Snowflake time logical type.
+   *
+   * @param physicalType Snowflake physical type of column
+   * @return column parquet primitive type name
+   */
+  private static PrimitiveType.PrimitiveTypeName getTimePrimitiveType(
+      AbstractRowBuffer.ColumnPhysicalType physicalType) {
+    PrimitiveType.PrimitiveTypeName type;
+    switch (physicalType) {
+      case SB4:
+        type = PrimitiveType.PrimitiveTypeName.INT32;
+        break;
+      case SB8:
+        type = PrimitiveType.PrimitiveTypeName.INT64;
+        break;
+      case SB16:
+        type = PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY;
+        break;
+      default:
+        throw new UnsupportedOperationException("Time physical type: " + physicalType);
+    }
+    return type;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
new file mode 100644
index 000000000..671af1a5b
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
@@ -0,0 +1,323 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.math.RoundingMode;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import javax.annotation.Nullable;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.parquet.schema.PrimitiveType;
+
+/** Parses a user column value into Parquet internal representation for buffering. */
+class ParquetValueParser {
+  /** Parquet internal value representation for buffering. */
+  static class ParquetBufferValue {
+    private final Object value;
+    private final float size;
+
+    ParquetBufferValue(Object value, float size) {
+      this.value = value;
+      this.size = size;
+    }
+
+    Object getValue() {
+      return value;
+    }
+
+    float getSize() {
+      return size;
+    }
+  }
+
+  /**
+   * Parses a user column value into Parquet internal representation for buffering.
+   *
+   * @param value column value provided by user in a row
+   * @param columnMetadata column metadata
+   * @param typeName Parquet primitive type name
+   * @param stats column stats to update
+   * @return parsed value and byte size of Parquet internal representation
+   */
+  static ParquetBufferValue parseColumnValueToParquet(
+      Object value,
+      ColumnMetadata columnMetadata,
+      PrimitiveType.PrimitiveTypeName typeName,
+      RowBufferStats stats) {
+    Utils.assertNotNull("Parquet column stats", stats);
+    float size = 0F;
+    if (value != null) {
+      AbstractRowBuffer.ColumnLogicalType logicalType =
+          AbstractRowBuffer.ColumnLogicalType.valueOf(columnMetadata.getLogicalType());
+      AbstractRowBuffer.ColumnPhysicalType physicalType =
+          AbstractRowBuffer.ColumnPhysicalType.valueOf(columnMetadata.getPhysicalType());
+      switch (typeName) {
+        case BOOLEAN:
+          int intValue = DataValidationUtil.validateAndParseBoolean(value);
+          value = intValue > 0;
+          stats.addIntValue(BigInteger.valueOf(intValue));
+          size = 1;
+          break;
+        case INT32:
+          int intVal =
+              getInt32Value(
+                  value,
+                  columnMetadata.getScale(),
+                  Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
+                  logicalType,
+                  physicalType);
+          value = intVal;
+          stats.addIntValue(BigInteger.valueOf(intVal));
+          size = 4;
+          break;
+        case INT64:
+          long longValue =
+              getInt64Value(
+                  value,
+                  columnMetadata.getScale(),
+                  Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
+                  logicalType,
+                  physicalType);
+          value = longValue;
+          stats.addIntValue(BigInteger.valueOf(longValue));
+          size = 8;
+          break;
+        case DOUBLE:
+          double doubleValue = DataValidationUtil.validateAndParseReal(value);
+          value = doubleValue;
+          stats.addRealValue(doubleValue);
+          size = 8;
+          break;
+        case BINARY:
+          String str = getBinaryValue(value, stats, columnMetadata);
+          value = str;
+          size = str.getBytes().length;
+          break;
+        case FIXED_LEN_BYTE_ARRAY:
+          BigInteger intRep =
+              getSb16Value(
+                  value,
+                  columnMetadata.getScale(),
+                  Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
+                  logicalType,
+                  physicalType);
+          stats.addIntValue(intRep);
+          value = getSb16Bytes(intRep);
+          size += 16;
+          break;
+        default:
+          throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
+      }
+    } else {
+      if (!columnMetadata.getNullable()) {
+        throw new SFException(
+            ErrorCode.INVALID_ROW, columnMetadata.getName(), "Passed null to non nullable field");
+      }
+      stats.incCurrentNullCount();
+    }
+    return new ParquetBufferValue(value, size);
+  }
+
+  /**
+   * Parses an int32 value based on Snowflake logical type.
+   *
+   * @param value column value provided by user in a row
+   * @param scale data type scale
+   * @param precision data type precision
+   * @param logicalType Snowflake logical type
+   * @param physicalType Snowflake physical type
+   * @return parsed int32 value
+   */
+  private static int getInt32Value(
+      Object value,
+      @Nullable Integer scale,
+      Integer precision,
+      AbstractRowBuffer.ColumnLogicalType logicalType,
+      AbstractRowBuffer.ColumnPhysicalType physicalType) {
+    int intVal;
+    switch (logicalType) {
+      case DATE:
+        intVal = DataValidationUtil.validateAndParseDate(value);
+        break;
+      case TIME:
+        Utils.assertNotNull("Unexpected null scale for TIME data type", scale);
+        intVal = DataValidationUtil.validateAndParseTime(value, scale).intValue();
+        break;
+      case FIXED:
+        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
+        bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
+        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
+        intVal = bigDecimalValue.intValue();
+        break;
+      default:
+        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
+    }
+    return intVal;
+  }
+
+  /**
+   * Parses an int64 value based on Snowflake logical type.
+   *
+   * @param value column value provided by user in a row
+   * @param scale data type scale
+   * @param precision data type precision
+   * @param logicalType Snowflake logical type
+   * @param physicalType Snowflake physical type
+   * @return parsed int64 value
+   */
+  private static long getInt64Value(
+      Object value,
+      int scale,
+      int precision,
+      AbstractRowBuffer.ColumnLogicalType logicalType,
+      AbstractRowBuffer.ColumnPhysicalType physicalType) {
+    long longValue;
+    switch (logicalType) {
+      case TIME:
+        Utils.assertNotNull("Unexpected null scale for TIME data type", scale);
+        longValue = DataValidationUtil.validateAndParseTime(value, scale).longValue();
+        break;
+      case TIMESTAMP_LTZ:
+      case TIMESTAMP_NTZ:
+        boolean ignoreTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
+        longValue =
+            DataValidationUtil.validateAndParseTimestampNtzSb16(value, scale, ignoreTimezone)
+                .getTimeInScale()
+                .longValue();
+        break;
+      case TIMESTAMP_TZ:
+        longValue =
+            DataValidationUtil.validateAndParseTimestampTz(value, scale)
+                .getSfTimestamp()
+                .orElseThrow(
+                    () ->
+                        new SFException(
+                            ErrorCode.INVALID_ROW,
+                            value,
+                            "Unable to parse timestamp for TIMESTAMP_TZ column"))
+                .toBinary(scale, true)
+                .longValue();
+        break;
+      case FIXED:
+        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
+        bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
+        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
+        longValue = bigDecimalValue.longValue();
+        break;
+      default:
+        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
+    }
+    return longValue;
+  }
+
+  /**
+   * Parses an int128 value based on Snowflake logical type.
+   *
+   * @param value column value provided by user in a row
+   * @param scale data type scale
+   * @param precision data type precision
+   * @param logicalType Snowflake logical type
+   * @param physicalType Snowflake physical type
+   * @return parsed int64 value
+   */
+  private static BigInteger getSb16Value(
+      Object value,
+      int scale,
+      int precision,
+      AbstractRowBuffer.ColumnLogicalType logicalType,
+      AbstractRowBuffer.ColumnPhysicalType physicalType) {
+    switch (logicalType) {
+      case TIMESTAMP_TZ:
+        return DataValidationUtil.validateAndParseTimestampTz(value, scale)
+            .getSfTimestamp()
+            .orElseThrow(
+                () ->
+                    new SFException(
+                        ErrorCode.INVALID_ROW,
+                        value,
+                        "Unable to parse timestamp for TIMESTAMP_TZ column"))
+            .toBinary(scale, true);
+      case TIMESTAMP_LTZ:
+      case TIMESTAMP_NTZ:
+        boolean ignoreTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
+        return DataValidationUtil.validateAndParseTimestampNtzSb16(value, scale, ignoreTimezone)
+            .getTimeInScale();
+      case FIXED:
+        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
+        // explicitly match the BigDecimal input scale with the Snowflake data type scale
+        bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
+        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
+        return bigDecimalValue.unscaledValue();
+      default:
+        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
+    }
+  }
+
+  /**
+   * Converts an int128 value to its byte array representation.
+   *
+   * @param intRep int128 value
+   * @return byte array representation
+   */
+  static byte[] getSb16Bytes(BigInteger intRep) {
+    byte[] bytes = intRep.toByteArray();
+    byte padByte = (byte) (bytes[0] < 0 ? -1 : 0);
+    byte[] bytesBE = new byte[16];
+    for (int i = 0; i < 16 - bytes.length; i++) {
+      bytesBE[i] = padByte;
+    }
+    System.arraycopy(bytes, 0, bytesBE, 16 - bytes.length, bytes.length);
+    return bytesBE;
+  }
+
+  /**
+   * Converts an object, string or binary value to its byte array representation.
+   *
+   * @param value value to parse
+   * @param stats column stats to update
+   * @param columnMetadata column metadata
+   * @return string (byte array) representation
+   */
+  private static String getBinaryValue(
+      Object value, RowBufferStats stats, ColumnMetadata columnMetadata) {
+    AbstractRowBuffer.ColumnLogicalType logicalType =
+        AbstractRowBuffer.ColumnLogicalType.valueOf(columnMetadata.getLogicalType());
+    String str;
+    if (logicalType.isObject()) {
+      switch (logicalType) {
+        case OBJECT:
+          str = DataValidationUtil.validateAndParseObject(value);
+          break;
+        case VARIANT:
+          str = DataValidationUtil.validateAndParseVariant(value);
+          break;
+        case ARRAY:
+          str = DataValidationUtil.validateAndParseArray(value);
+          break;
+        default:
+          throw new SFException(
+              ErrorCode.UNKNOWN_DATA_TYPE, logicalType, columnMetadata.getPhysicalType());
+      }
+    } else if (logicalType == AbstractRowBuffer.ColumnLogicalType.BINARY) {
+      String maxLengthString = columnMetadata.getLength().toString();
+      byte[] bytes =
+          DataValidationUtil.validateAndParseBinary(
+              value, Optional.of(maxLengthString).map(Integer::parseInt));
+      str = new String(bytes, StandardCharsets.UTF_8);
+      stats.addStrValue(str);
+    } else {
+      String maxLengthString = columnMetadata.getLength().toString();
+      str =
+          DataValidationUtil.validateAndParseString(
+              value, Optional.of(maxLengthString).map(Integer::parseInt));
+      stats.addStrValue(str);
+    }
+    return str;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index 739ae5499..44b90987b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -25,7 +25,8 @@
  * Register one or more blobs to the targeted Snowflake table, it will be done using the dedicated
  * thread in order to maintain ordering per channel
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 class RegisterService<T> {
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
index a229f4db3..2aa9d9861 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
@@ -13,7 +13,8 @@
  * Interface for the buffer in the Streaming Ingest channel that holds the un-flushed rows, these
  * rows will be converted to the underlying format implementation for faster processing
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 interface RowBuffer<T> {
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index d2d64db29..7016b23f5 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -30,7 +30,8 @@
 /**
  * The first version of implementation for SnowflakeStreamingIngestChannel
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIngestChannel {
 
@@ -148,9 +149,20 @@ private RowBuffer<T> createRowBuffer(Constants.BdecVersion bdecVersion) {
     // TODO: The circular dependency SnowflakeStreamingIngestChannelInternal <-> RowBuffer
     // (SNOW-657667)
     // can be probably reconsidered
-    //noinspection unchecked
-    return (RowBuffer<T>)
-        new ArrowRowBuffer((SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>) this);
+    switch (bdecVersion) {
+      case ONE:
+      case TWO:
+        //noinspection unchecked
+        return (RowBuffer<T>)
+            new ArrowRowBuffer((SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>) this);
+      case THREE:
+        //noinspection unchecked
+        return (RowBuffer<T>)
+            new ParquetRowBuffer((SnowflakeStreamingIngestChannelInternal<ParquetChunkData>) this);
+      default:
+        throw new SFException(
+            ErrorCode.INTERNAL_ERROR, "Unsupported BDEC format version: " + bdecVersion);
+    }
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index bb868a847..169be2403 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -82,7 +82,8 @@
  * <li>the channel cache, which contains all the channels that belong to this account
  * <li>the flush service, which schedules and coordinates the flush to Snowflake tables
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStreamingIngestClient {
 
@@ -550,7 +551,7 @@ List<BlobMetadata> getRetryBlobs(
                   .collect(Collectors.toList());
           if (!relevantChunks.isEmpty()) {
             retryBlobs.add(
-                new BlobMetadata(
+                BlobMetadata.createBlobMetadata(
                     blobMetadata.getPath(),
                     blobMetadata.getMD5(),
                     blobMetadata.getVersion(),
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index fed4022e1..68602e0a9 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -83,7 +83,14 @@ public enum BdecVersion {
     ONE(1),
 
     /** Uses Arrow to generate BDEC chunks with {@link ArrowBatchWriteMode#FILE}. */
-    TWO(2);
+    TWO(2),
+
+    /**
+     * Uses Parquet to generate BDEC chunks with {@link
+     * net.snowflake.ingest.streaming.internal.ParquetRowBuffer} (page-level compression). This
+     * version is experimental and WIP at the moment.
+     */
+    THREE(3);
 
     private final byte version;
 
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 5e5e594cd..9f586a04c 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -14,7 +14,6 @@
 import static net.snowflake.ingest.utils.Constants.USER;
 import static net.snowflake.ingest.utils.Constants.WAREHOUSE;
 import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION;
-import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -35,6 +34,7 @@
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.Utils;
 import org.apache.commons.codec.binary.Base64;
 import org.junit.Assert;
@@ -180,7 +180,7 @@ public static KeyPair getKeyPair() throws Exception {
     return keyPair;
   }
 
-  public static Properties getProperties() throws Exception {
+  public static Properties getProperties(Constants.BdecVersion bdecVersion) throws Exception {
     if (profile == null) {
       init();
     }
@@ -195,16 +195,10 @@ public static Properties getProperties() throws Exception {
     props.put(PRIVATE_KEY, privateKeyPem);
     props.put(ROLE, role);
     props.put(ACCOUNT_URL, getAccountURL());
-    props.put(BLOB_FORMAT_VERSION, getBlobFormatVersion());
+    props.put(BLOB_FORMAT_VERSION, bdecVersion.toByte());
     return props;
   }
 
-  private static byte getBlobFormatVersion() {
-    return profile.has(BLOB_FORMAT_VERSION)
-        ? (byte) profile.get(BLOB_FORMAT_VERSION).asInt()
-        : BLOB_FORMAT_VERSION_DEFAULT.toByte();
-  }
-
   /**
    * Create snowflake jdbc connection
    *
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
index c07966c68..5785bd221 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
@@ -57,15 +57,13 @@ public void testFieldNumberAfterFlush() {
   @Test
   public void buildFieldFixedSB1() {
     // FIXED, SB1
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB1");
-    testCol.setNullable(true);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB1")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -80,15 +78,13 @@ public void buildFieldFixedSB1() {
 
   @Test
   public void buildFieldFixedSB2() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB2");
-    testCol.setNullable(false);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB2")
+            .nullable(false)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -103,15 +99,13 @@ public void buildFieldFixedSB2() {
 
   @Test
   public void buildFieldFixedSB4() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB4");
-    testCol.setNullable(true);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB4")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -125,15 +119,13 @@ public void buildFieldFixedSB4() {
 
   @Test
   public void buildFieldFixedSB8() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -147,15 +139,13 @@ public void buildFieldFixedSB8() {
 
   @Test
   public void buildFieldFixedSB16() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB16");
-    testCol.setNullable(true);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     ArrowType expectedType =
@@ -172,15 +162,13 @@ public void buildFieldFixedSB16() {
 
   @Test
   public void buildFieldLobVariant() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("LOB");
-    testCol.setNullable(true);
-    testCol.setLogicalType("VARIANT");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("VARIANT")
+            .physicalType("LOB")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -194,15 +182,13 @@ public void buildFieldLobVariant() {
 
   @Test
   public void buildFieldTimestampNtzSB8() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIMESTAMP_NTZ");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -216,15 +202,13 @@ public void buildFieldTimestampNtzSB8() {
 
   @Test
   public void buildFieldTimestampNtzSB16() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB16");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIMESTAMP_NTZ");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -242,15 +226,13 @@ public void buildFieldTimestampNtzSB16() {
 
   @Test
   public void buildFieldTimestampTzSB8() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIMESTAMP_TZ");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_TZ")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -268,15 +250,13 @@ public void buildFieldTimestampTzSB8() {
 
   @Test
   public void buildFieldTimestampTzSB16() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB16");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIMESTAMP_TZ");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_TZ")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -295,16 +275,14 @@ public void buildFieldTimestampTzSB16() {
   }
 
   @Test
-  public void buildFieldDate() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("DATE");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+  public void buildFieldTimestampDate() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("DATE")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -318,15 +296,13 @@ public void buildFieldDate() {
 
   @Test
   public void buildFieldTimeSB4() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB4");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIME");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB4")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -340,15 +316,13 @@ public void buildFieldTimeSB4() {
 
   @Test
   public void buildFieldTimeSB8() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIME");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -362,15 +336,13 @@ public void buildFieldTimeSB8() {
 
   @Test
   public void buildFieldBoolean() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("BINARY");
-    testCol.setNullable(true);
-    testCol.setLogicalType("BOOLEAN");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("BOOLEAN")
+            .physicalType("BINARY")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -384,15 +356,13 @@ public void buildFieldBoolean() {
 
   @Test
   public void buildFieldRealSB16() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB16");
-    testCol.setNullable(true);
-    testCol.setLogicalType("REAL");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("REAL")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java b/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
new file mode 100644
index 000000000..51d4226bb
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
@@ -0,0 +1,164 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+/** Helper class to build ColumnMetadata */
+public class ColumnMetadataBuilder {
+  private String name;
+  private String type;
+  private String logicalType;
+  private String physicalType;
+  private Integer precision;
+  private Integer scale;
+  private Integer byteLength;
+  private Integer length;
+  private boolean nullable;
+  private String collation;
+
+  /**
+   * Returns a new ColumnMetadata builder
+   *
+   * @return builder
+   */
+  public static ColumnMetadataBuilder newBuilder() {
+    ColumnMetadataBuilder mb = new ColumnMetadataBuilder();
+    mb.name = "testCol";
+    mb.byteLength = 14;
+    mb.length = 11;
+    mb.scale = 0;
+    mb.precision = 4;
+    return mb;
+  }
+
+  /**
+   * Set name
+   *
+   * @param name column name
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder name(String name) {
+    this.name = name;
+    return this;
+  }
+
+  /**
+   * Set type
+   *
+   * @param type type (as defined in ColumnMetadata)
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder type(String type) {
+    this.type = type;
+    return this;
+  }
+
+  /**
+   * Set column logical type
+   *
+   * @param logicalType logical type
+   * @return columnMetadata object
+   */
+  public ColumnMetadataBuilder logicalType(String logicalType) {
+    this.logicalType = logicalType;
+    return this;
+  }
+
+  /**
+   * Set column physical type
+   *
+   * @param physicalType physical type
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder physicalType(String physicalType) {
+    this.physicalType = physicalType;
+    return this;
+  }
+
+  /**
+   * Set column precision
+   *
+   * @param precision precision
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder precision(int precision) {
+    this.precision = precision;
+    return this;
+  }
+
+  /**
+   * Set column scale
+   *
+   * @param scale scale
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder scale(int scale) {
+    this.scale = scale;
+    return this;
+  }
+
+  /**
+   * Set column length in bytes
+   *
+   * @param byteLength length in bytes
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder byteLength(int byteLength) {
+    this.byteLength = byteLength;
+    return this;
+  }
+
+  /**
+   * Set column length (in chars)
+   *
+   * @param length length
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder length(int length) {
+    this.length = length;
+    return this;
+  }
+
+  /**
+   * Set column nullability
+   *
+   * @param nullable nullable
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder nullable(boolean nullable) {
+    this.nullable = nullable;
+    return this;
+  }
+
+  /**
+   * Set column collation
+   *
+   * @param collation collation
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder collation(String collation) {
+    this.collation = collation;
+    return this;
+  }
+
+  /**
+   * Build a columnMetadata object from the set values
+   *
+   * @return columnMetadata object
+   */
+  public ColumnMetadata build() {
+    ColumnMetadata colMetadata = new ColumnMetadata();
+    colMetadata.setName(name);
+    colMetadata.setType(type);
+    colMetadata.setPhysicalType(physicalType);
+    colMetadata.setNullable(nullable);
+    colMetadata.setLogicalType(logicalType);
+    colMetadata.setByteLength(byteLength);
+    colMetadata.setLength(length);
+    colMetadata.setScale(scale);
+    colMetadata.setPrecision(precision);
+    colMetadata.setCollation(collation);
+    return colMetadata;
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 80103e2df..540cd5ea8 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -55,7 +55,8 @@
 public class FlushServiceTest {
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> testContextFactory() {
-    return Arrays.asList(new Object[][] {{ArrowTestContext.createFactory()}});
+    return Arrays.asList(
+        new Object[][] {{ArrowTestContext.createFactory()}, {ParquetTestContext.createFactory()}});
   }
 
   public FlushServiceTest(TestContextFactory<?> testContextFactory) {
@@ -182,11 +183,6 @@ ChannelBuilder setEncryptionKeyId(Long encryptionKeyId) {
         return this;
       }
 
-      ChannelBuilder setOnErrorOption(OpenChannelRequest.OnErrorOption onErrorOption) {
-        this.onErrorOption = onErrorOption;
-        return this;
-      }
-
       SnowflakeStreamingIngestChannelInternal<T> buildAndAdd() {
         SnowflakeStreamingIngestChannelInternal<T> channel =
             createChannel(
@@ -285,6 +281,48 @@ TestContext<VectorSchemaRoot> create() {
     }
   }
 
+  private static class ParquetTestContext extends TestContext<List<List<Object>>> {
+
+    SnowflakeStreamingIngestChannelInternal<List<List<Object>>> createChannel(
+        String name,
+        String dbName,
+        String schemaName,
+        String tableName,
+        String offsetToken,
+        Long channelSequencer,
+        Long rowSequencer,
+        String encryptionKey,
+        Long encryptionKeyId,
+        OpenChannelRequest.OnErrorOption onErrorOption) {
+      return new SnowflakeStreamingIngestChannelInternal<>(
+          name,
+          dbName,
+          schemaName,
+          tableName,
+          offsetToken,
+          channelSequencer,
+          rowSequencer,
+          client,
+          encryptionKey,
+          encryptionKeyId,
+          onErrorOption,
+          Constants.BdecVersion.THREE,
+          null);
+    }
+
+    @Override
+    public void close() {}
+
+    static TestContextFactory<List<List<Object>>> createFactory() {
+      return new TestContextFactory<List<List<Object>>>("Parquet") {
+        @Override
+        TestContext<List<List<Object>>> create() {
+          return new ParquetTestContext();
+        }
+      };
+    }
+  }
+
   TestContextFactory<?> testContextFactory;
   TestContext<?> testContext;
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
new file mode 100644
index 000000000..a477ac977
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
@@ -0,0 +1,480 @@
+package net.snowflake.ingest.streaming.internal;
+
+import java.util.Map;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ParquetTypeGeneratorTest {
+
+  @Test
+  public void buildFieldFixedSB1() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB1")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(4)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
+        .expectedLogicalTypeAnnotation(
+            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB1.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldFixedSB2() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB2")
+            .nullable(false)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(4)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
+        .expectedLogicalTypeAnnotation(
+            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
+        .expectedRepetition(Type.Repetition.REQUIRED)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB2.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldFixedSB4() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB4")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(4)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
+        .expectedLogicalTypeAnnotation(
+            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB4.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldFixedSB8() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(8)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
+        .expectedLogicalTypeAnnotation(
+            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldFixedSB16() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(16)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+        .expectedLogicalTypeAnnotation(
+            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldLobVariant() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("VARIANT")
+            .physicalType("LOB")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(0)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.BINARY)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.stringType())
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.VARIANT.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.LOB.getOrdinal())
+        .assertMatches();
+
+    Assert.assertEquals("1", typeInfo.getMetadata().get(0 + ":obj_enc"));
+  }
+
+  @Test
+  public void buildFieldTimestampNtzSB8() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(8)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldTimestampNtzSB16() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(16)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 38))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldTimestampTzSB8() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_TZ")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(8)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_TZ.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldTimestampTzSB16() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_TZ")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(16)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 38))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_TZ.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldDate() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("DATE")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(0)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.dateType())
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.DATE.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldTimeSB4() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB4")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(4)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 9))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIME.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB4.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldTimeSB8() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(8)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIME.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldBoolean() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("BOOLEAN")
+            .physicalType("BINARY")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(0)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.BOOLEAN)
+        .expectedLogicalTypeAnnotation(null)
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.BOOLEAN.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.BINARY.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldRealSB16() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("REAL")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(0)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.DOUBLE)
+        .expectedLogicalTypeAnnotation(null)
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.REAL.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
+        .assertMatches();
+  }
+
+  /** Builder that helps to assert parquet type info */
+  private static class ParquetTypeInfoAssertionBuilder {
+    private String fieldName;
+    private int fieldId;
+    private PrimitiveType.PrimitiveTypeName primitiveTypeName;
+    private LogicalTypeAnnotation logicalTypeAnnotation;
+    private Type.Repetition repetition;
+    private int typeLength;
+    private String colMetadata;
+    private ParquetTypeGenerator.ParquetTypeInfo typeInfo;
+
+    static ParquetTypeInfoAssertionBuilder newBuilder() {
+      ParquetTypeInfoAssertionBuilder builder = new ParquetTypeInfoAssertionBuilder();
+      return builder;
+    }
+
+    ParquetTypeInfoAssertionBuilder typeInfo(ParquetTypeGenerator.ParquetTypeInfo typeInfo) {
+      this.typeInfo = typeInfo;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedFieldName(String fieldName) {
+      this.fieldName = fieldName;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedFieldId(int fieldId) {
+      this.fieldId = fieldId;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedPrimitiveTypeName(
+        PrimitiveType.PrimitiveTypeName primitiveTypeName) {
+      this.primitiveTypeName = primitiveTypeName;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedLogicalTypeAnnotation(
+        LogicalTypeAnnotation logicalTypeAnnotation) {
+      this.logicalTypeAnnotation = logicalTypeAnnotation;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedRepetition(Type.Repetition repetition) {
+      this.repetition = repetition;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedTypeLength(int typeLength) {
+      this.typeLength = typeLength;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedColMetaData(String colMetaData) {
+      this.colMetadata = colMetaData;
+      return this;
+    }
+
+    void assertMatches() {
+      Type type = typeInfo.getParquetType();
+      Map<String, String> metadata = typeInfo.getMetadata();
+      Assert.assertEquals(fieldName, type.getName());
+      Assert.assertEquals(typeLength, type.asPrimitiveType().getTypeLength());
+      Assert.assertEquals(fieldId, type.asPrimitiveType().getId().intValue());
+
+      Assert.assertEquals(primitiveTypeName, type.asPrimitiveType().getPrimitiveTypeName());
+      Assert.assertEquals(logicalTypeAnnotation, type.getLogicalTypeAnnotation());
+      Assert.assertEquals(repetition, type.getRepetition());
+      Assert.assertEquals(colMetadata, metadata.get(type.getId().toString()));
+    }
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
new file mode 100644
index 000000000..3dc313dc6
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
@@ -0,0 +1,570 @@
+package net.snowflake.ingest.streaming.internal;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.Map;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.parquet.schema.PrimitiveType;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ParquetValueParserTest {
+
+  @Test
+  public void parseValueFixedSB1ToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB1")
+            .scale(0)
+            .precision(2)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            12, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Integer.class)
+        .expectedParsedValue(12)
+        .expectedSize(4.0f)
+        .expectedMinMax(BigInteger.valueOf(12))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueFixedSB2ToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB2")
+            .scale(0)
+            .precision(4)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            1234, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Integer.class)
+        .expectedParsedValue(1234)
+        .expectedSize(4.0f)
+        .expectedMinMax(BigInteger.valueOf(1234))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueFixedSB4ToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB4")
+            .scale(0)
+            .precision(9)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            123456789, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Integer.class)
+        .expectedParsedValue(123456789)
+        .expectedSize(4.0f)
+        .expectedMinMax(BigInteger.valueOf(123456789))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueFixedSB8ToInt64() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB8")
+            .scale(0)
+            .precision(18)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            123456789987654321L, testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Long.class)
+        .expectedParsedValue(123456789987654321L)
+        .expectedSize(8.0f)
+        .expectedMinMax(BigInteger.valueOf(123456789987654321L))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueFixedSB16ToByteArray() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB16")
+            .scale(0)
+            .precision(38)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            new BigDecimal("91234567899876543219876543211234567891"),
+            testCol,
+            PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
+            rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(byte[].class)
+        .expectedParsedValue(
+            ParquetValueParser.getSb16Bytes(
+                new BigInteger("91234567899876543219876543211234567891")))
+        .expectedSize(16.0f)
+        .expectedMinMax(new BigInteger("91234567899876543219876543211234567891"))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueFixedDecimalToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB8")
+            .scale(5)
+            .precision(10)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            new BigDecimal("12345.54321"),
+            testCol,
+            PrimitiveType.PrimitiveTypeName.DOUBLE,
+            rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Double.class)
+        .expectedParsedValue(Double.valueOf("12345.54321"))
+        .expectedSize(8.0f)
+        .expectedMinMax(Double.valueOf("12345.54321"))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueDouble() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("REAL")
+            .physicalType("DOUBLE")
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            12345.54321d, testCol, PrimitiveType.PrimitiveTypeName.DOUBLE, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Double.class)
+        .expectedParsedValue(Double.valueOf(12345.54321))
+        .expectedSize(8.0f)
+        .expectedMinMax(Double.valueOf(12345.54321))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueBoolean() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("BOOLEAN")
+            .physicalType("SB1")
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            true, testCol, PrimitiveType.PrimitiveTypeName.BOOLEAN, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Boolean.class)
+        .expectedParsedValue(true)
+        .expectedSize(1.0f)
+        .expectedMinMax(BigInteger.valueOf(1))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueBinary() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("BINARY")
+            .physicalType("LOB")
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "Length7".getBytes(), testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(String.class)
+        .expectedParsedValue("Length7")
+        .expectedSize(7.0f)
+        .expectedMinMax("Length7")
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueVariantToBinary() {
+    testJsonWithLogicalType("VARIANT");
+  }
+
+  @Test
+  public void parseValueObjectToBinary() {
+    testJsonWithLogicalType("OBJECT");
+  }
+
+  private void testJsonWithLogicalType(String logicalType) {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType(logicalType)
+            .physicalType("BINARY")
+            .nullable(true)
+            .build();
+
+    String var =
+        "{\"key1\":-879869596,\"key2\":\"value2\",\"key3\":null,"
+            + "\"key4\":{\"key41\":0.032437,\"key42\":\"value42\",\"key43\":null}}";
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            var, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(String.class)
+        .expectedParsedValue(var)
+        .expectedSize(var.getBytes().length)
+        .expectedMinMax(null)
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueArrayToBinary() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("ARRAY")
+            .physicalType("BINARY")
+            .nullable(true)
+            .build();
+
+    Map<String, String> input = new HashMap<>();
+    input.put("a", "1");
+    input.put("b", "2");
+    input.put("c", "3");
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            input, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
+
+    String resultArray = "[{\"a\":\"1\",\"b\":\"2\",\"c\":\"3\"}]";
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(String.class)
+        .expectedParsedValue(resultArray)
+        .expectedSize(resultArray.length())
+        .expectedMinMax(null)
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueTimestampNtzSB4Error() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB4")
+            .scale(0) // seconds
+            .precision(9)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    SFException exception =
+        Assert.assertThrows(
+            SFException.class,
+            () ->
+                ParquetValueParser.parseColumnValueToParquet(
+                    "2013-04-28 20:57:00",
+                    testCol,
+                    PrimitiveType.PrimitiveTypeName.INT32,
+                    rowBufferStats));
+    Assert.assertEquals(
+        "Unknown data type for logical: TIMESTAMP_NTZ, physical: SB4.", exception.getMessage());
+  }
+
+  @Test
+  public void parseValueTimestampNtzSB8ToINT64() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB8")
+            .scale(3) // millis
+            .precision(18)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "2013-04-28 20:57:01.000",
+            testCol,
+            PrimitiveType.PrimitiveTypeName.INT64,
+            rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Long.class)
+        .expectedParsedValue(1367182621000L)
+        .expectedSize(8.0f)
+        .expectedMinMax(BigInteger.valueOf(1367182621000L))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueTimestampNtzSB16ToByteArray() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB16")
+            .nullable(true)
+            .scale(9) // nanos
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "2022-09-18 22:05:07.123456789",
+            testCol,
+            PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
+            rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(byte[].class)
+        .expectedParsedValue(
+            ParquetValueParser.getSb16Bytes(BigInteger.valueOf(1663538707123456789L)))
+        .expectedSize(16.0f)
+        .expectedMinMax(BigInteger.valueOf(1663538707123456789L))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueDateToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("DATE")
+            .physicalType("SB4")
+            .scale(0) // seconds
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "2021-01-01", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Integer.class)
+        .expectedParsedValue(Integer.valueOf(18628))
+        .expectedSize(4.0f)
+        .expectedMinMax(BigInteger.valueOf(18628))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueTimeSB4ToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB4")
+            .scale(0) // seconds
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "01:00:00", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Integer.class)
+        .expectedParsedValue(3600)
+        .expectedSize(4.0f)
+        .expectedMinMax(BigInteger.valueOf(3600))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueTimeSB8ToInt64() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB8")
+            .scale(3) // milliseconds
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "01:00:00.123", testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Long.class)
+        .expectedParsedValue(3600123L)
+        .expectedSize(8.0f)
+        .expectedMinMax(BigInteger.valueOf(3600123))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueTimeSB16Error() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB16")
+            .scale(9) // nanos
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    SFException exception =
+        Assert.assertThrows(
+            SFException.class,
+            () ->
+                ParquetValueParser.parseColumnValueToParquet(
+                    "11:00:00.12345678",
+                    testCol,
+                    PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
+                    rowBufferStats));
+    Assert.assertEquals(
+        "Unknown data type for logical: TIME, physical: SB16.", exception.getMessage());
+  }
+
+  /** Builder that helps to assert parsing of values to parquet types */
+  private static class ParquetValueParserAssertionBuilder {
+    private ParquetValueParser.ParquetBufferValue parquetBufferValue;
+    private RowBufferStats rowBufferStats;
+    private Class valueClass;
+    private Object value;
+    private float size;
+    private Object minMaxStat;
+
+    static ParquetValueParserAssertionBuilder newBuilder() {
+      ParquetValueParserAssertionBuilder builder = new ParquetValueParserAssertionBuilder();
+      return builder;
+    }
+
+    ParquetValueParserAssertionBuilder parquetBufferValue(
+        ParquetValueParser.ParquetBufferValue parquetBufferValue) {
+      this.parquetBufferValue = parquetBufferValue;
+      return this;
+    }
+
+    ParquetValueParserAssertionBuilder rowBufferStats(RowBufferStats rowBufferStats) {
+      this.rowBufferStats = rowBufferStats;
+      return this;
+    }
+
+    ParquetValueParserAssertionBuilder expectedValueClass(Class valueClass) {
+      this.valueClass = valueClass;
+      return this;
+    }
+
+    ParquetValueParserAssertionBuilder expectedParsedValue(Object value) {
+      this.value = value;
+      return this;
+    }
+
+    ParquetValueParserAssertionBuilder expectedSize(float size) {
+      this.size = size;
+      return this;
+    }
+
+    public ParquetValueParserAssertionBuilder expectedMinMax(Object minMaxStat) {
+      this.minMaxStat = minMaxStat;
+      return this;
+    }
+
+    void assertMatches() {
+      Assert.assertEquals(valueClass, parquetBufferValue.getValue().getClass());
+      System.out.println("parquetBufferValue = " + parquetBufferValue.getValue().toString());
+      if (valueClass.equals(byte[].class)) {
+        Assert.assertArrayEquals((byte[]) value, (byte[]) parquetBufferValue.getValue());
+      } else {
+        Assert.assertEquals(value, parquetBufferValue.getValue());
+      }
+      Assert.assertEquals(size, parquetBufferValue.getSize(), 0);
+      if (minMaxStat instanceof BigInteger) {
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinIntValue());
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxIntValue());
+        return;
+      } else if (minMaxStat instanceof String || valueClass.equals(String.class)) {
+        // String can have null min/max stats for variant data types
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinColStrValue());
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxColStrValue());
+        return;
+      } else if (minMaxStat instanceof Double || minMaxStat instanceof BigDecimal) {
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinRealValue());
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxRealValue());
+        return;
+      }
+      throw new IllegalArgumentException("Unknown data type for min stat");
+    }
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 7bd986e8a..8d28c0cb9 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -26,7 +26,11 @@
 public class RowBufferTest {
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> bdecVersion() {
-    return Arrays.asList(new Object[][] {{"Arrow", Constants.BdecVersion.ONE}});
+    return Arrays.asList(
+        new Object[][] {
+          {"Arrow", Constants.BdecVersion.ONE},
+          {"Parquet", Constants.BdecVersion.THREE}
+        });
   }
 
   private final Constants.BdecVersion bdecVersion;
@@ -1029,6 +1033,7 @@ private void testE2EBinaryHelper(OpenChannelRequest.OnErrorOption onErrorOption)
     colBinary.setPhysicalType("LOB");
     colBinary.setNullable(true);
     colBinary.setLogicalType("BINARY");
+    colBinary.setLength(32);
     colBinary.setScale(0);
 
     innerBuffer.setupSchema(Collections.singletonList(colBinary));
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 175db5fb8..c5189bcde 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -9,14 +9,15 @@
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
-import java.util.Random;
 import java.util.TimeZone;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
@@ -31,6 +32,7 @@
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.ParameterProvider;
 import net.snowflake.ingest.utils.SFException;
@@ -38,9 +40,23 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 /** Example streaming ingest sdk integration test */
+@RunWith(Parameterized.class)
 public class StreamingIngestIT {
+  @Parameterized.Parameters(name = "{0}")
+  public static Collection<Object[]> bdecVersion() {
+    return Arrays.asList(
+        new Object[][] {
+          {"Arrow", Constants.BdecVersion.ONE},
+          // TODO: uncomment once SNOW-659721 is deployed and we set the parameter
+          // DISABLE_PARQUET_CACHE to true for the test account
+          // {"Parquet", Constants.BdecVersion.THREE}
+        });
+  }
+
   private static final String TEST_TABLE = "STREAMING_INGEST_TEST_TABLE";
   private static final String TEST_DB_PREFIX = "STREAMING_INGEST_TEST_DB";
   private static final String TEST_SCHEMA = "STREAMING_INGEST_TEST_SCHEMA";
@@ -55,11 +71,23 @@ public class StreamingIngestIT {
   private Connection jdbcConnection;
   private String testDb;
 
+  private final Constants.BdecVersion bdecVersion;
+
+  public StreamingIngestIT(
+      @SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
+    this.bdecVersion = bdecVersion;
+  }
+
   @Before
   public void beforeAll() throws Exception {
     testDb = TEST_DB_PREFIX + "_" + UUID.randomUUID().toString().substring(0, 4);
     // Create a streaming ingest client
     jdbcConnection = TestUtils.getConnection(true);
+    if (bdecVersion == Constants.BdecVersion.THREE) {
+      // TODO: encryption and interleaved mode are not yet supported by server side's Parquet
+      // scanner if local file cache is enabled (SNOW-656500)
+      jdbcConnection.createStatement().execute("alter session set disable_parquet_cache=true;");
+    }
     jdbcConnection
         .createStatement()
         .execute(String.format("create or replace database %s;", testDb));
@@ -78,7 +106,7 @@ public void beforeAll() throws Exception {
         .createStatement()
         .execute(String.format("use warehouse %s", TestUtils.getWarehouse()));
 
-    prop = TestUtils.getProperties();
+    prop = TestUtils.getProperties(bdecVersion);
     if (prop.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       prop.setProperty(ROLE, "ACCOUNTADMIN");
     }
@@ -1042,7 +1070,7 @@ private void verifyTableRowCount(int rowNumber, String tableName) {
   }
 
   @Test
-  public void testDataTypes() {
+  public void testDataTypes() throws SQLException, ParseException {
     String tableName = "t_data_types";
     try {
       jdbcConnection
@@ -1080,94 +1108,195 @@ public void testDataTypes() {
 
     SnowflakeStreamingIngestChannel channel = openChannel(tableName, "CHANNEL");
 
-    Random r = new Random();
-    for (int val = 0; val < 10000; val++) {
-      verifyInsertValidationResponse(channel.insertRow(getRandomRow(r), Integer.toString(val)));
-    }
-    waitChannelFlushed(channel, 10000);
+    Map<String, Object> negRow = getNegRow();
+    verifyInsertValidationResponse(channel.insertRow(negRow, Integer.toString(0)));
+
+    Map<String, Object> nullRow = getNullRow();
+    verifyInsertValidationResponse(channel.insertRow(nullRow, Integer.toString(1)));
+
+    Map<String, Object> posRow = getPosRow();
+    verifyInsertValidationResponse(channel.insertRow(posRow, Integer.toString(2)));
+
+    waitChannelFlushed(channel, 3);
+    verifyTableRowCount(3, tableName);
+
+    ResultSet result =
+        jdbcConnection
+            .createStatement()
+            .executeQuery(
+                String.format(
+                    "select * from %s.%s.%s order by num", testDb, TEST_SCHEMA, tableName));
+
+    result.next(); // negRow
+    assertNegRow(negRow, result);
+
+    result.next(); // nullRow
+    assertNullRow(nullRow, result);
+
+    result.next(); // posRow
+    assertPosRow(posRow, result);
   }
 
-  private static Map<String, Object> getRandomRow(Random r) {
-    Map<String, Object> row = new HashMap<>();
-    row.put("num", r.nextInt());
-    row.put("num_10_5", nextFloat(r));
-    row.put(
-        "num_38_0",
-        new BigDecimal("" + nextLongOfPrecision(r, 18) + Math.abs(nextLongOfPrecision(r, 18))));
-    row.put("num_2_0", r.nextInt(100));
-    row.put("num_4_0", r.nextInt(10000));
-    row.put("num_9_0", r.nextInt(1000000000));
-    row.put("num_18_0", nextLongOfPrecision(r, 18));
-    row.put("num_float", nextFloat(r));
-    row.put("str_varchar", nextString(r));
-    row.put("bin", nextBytes(r));
-    row.put("bl", r.nextBoolean());
-    row.put("var", nextJson(r));
-    row.put("obj", nextJson(r));
-    row.put("arr", Arrays.asList(r.nextInt(100), r.nextInt(100), r.nextInt(100)));
-    row.put(
-        "epochdays",
-        String.valueOf(Math.abs(r.nextInt()) % (18963 * 24 * 60 * 60))); // DATE, 02.12.2021
-    row.put(
-        "timesec",
-        String.valueOf(
-            (r.nextInt(11) * 60 * 60 + r.nextInt(59) * 60 + r.nextInt(59)) * 10000
-                + r.nextInt(9999))); // TIME(4), 05:12:43.4536
-    row.put(
-        "timenano",
-        String.valueOf(
-            (14 * 60 * 60 + 26 * 60 + 34) * 1000000000L
-                + 437582643)); // TIME(9), 14:26:34.437582643
-    row.put(
-        "epochsec",
-        String.valueOf(
-            Math.abs(r.nextLong()) % 1638459438)); // TIMESTAMP_LTZ(0), 02.12.2021 15:37:18
-    row.put(
-        "epochnano",
-        String.format(
-            "%d%d",
-            Math.abs(r.nextInt()) % 1999999999,
-            100000000
-                + Math.abs(
-                    r.nextInt(899999999)))); // TIMESTAMP_LTZ(9), 18.05.2033 03:33:19.999999999
-    return row;
+  private Map<String, Object> getNegRow() {
+    Map<String, Object> negRow = new HashMap<>();
+    negRow.put("num", -123);
+    negRow.put("num_10_5", new BigDecimal("-12345.54321"));
+    negRow.put("num_38_0", new BigDecimal("-91234567899876543219876543211234567891"));
+    negRow.put("num_2_0", -12);
+    negRow.put("num_4_0", -1234);
+    negRow.put("num_9_0", -123456789);
+    negRow.put("num_18_0", -123456789987654321L);
+    negRow.put("num_float", -12345.54321f);
+    negRow.put("str_varchar", "zxccvbnm,.");
+    negRow.put("bin", new byte[] {1, 2, 3});
+    negRow.put("bl", true);
+    negRow.put("var", "{}");
+    negRow.put("obj", "{}");
+    negRow.put("arr", Arrays.asList());
+    negRow.put("epochdays", "0");
+    negRow.put("epochsec", "0");
+    negRow.put("epochnano", "0");
+    negRow.put("timesec", "0");
+    negRow.put("timenano", "0");
+
+    return negRow;
   }
 
-  private static long nextLongOfPrecision(Random r, int precision) {
-    return r.nextLong() % Math.round(Math.pow(10, precision));
+  private Map<String, Object> getNullRow() {
+    Map<String, Object> nullRow = new HashMap<>();
+    nullRow.put("num", 0);
+    nullRow.put("num_10_5", new BigDecimal("0.00000"));
+    nullRow.put("num_38_0", new BigDecimal("0"));
+    nullRow.put("num_2_0", 0);
+    nullRow.put("num_4_0", 0);
+    nullRow.put("num_9_0", 0);
+    nullRow.put("num_18_0", 0L);
+    nullRow.put("num_float", 0.0f);
+    nullRow.put("str_varchar", null);
+    nullRow.put("bin", null);
+    nullRow.put("bl", false);
+    nullRow.put("var", null);
+    nullRow.put("obj", null);
+    nullRow.put("arr", null);
+    nullRow.put("epochdays", null);
+    nullRow.put("epochsec", null);
+    nullRow.put("epochnano", null);
+    nullRow.put("timesec", null);
+    nullRow.put("timenano", null);
+
+    return nullRow;
   }
 
-  private static String nextString(Random r) {
-    return new String(nextBytes(r));
+  private Map<String, Object> getPosRow() {
+    Map<String, Object> posRow = new HashMap<>();
+    posRow.put("num", 123);
+    posRow.put("num_10_5", new BigDecimal("12345.54321"));
+    posRow.put("num_38_0", new BigDecimal("91234567899876543219876543211234567891"));
+    posRow.put("num_2_0", 12);
+    posRow.put("num_4_0", 1234);
+    posRow.put("num_9_0", 123456789);
+    posRow.put("num_18_0", 123456789987654321L);
+    posRow.put("num_float", 12345.54321f);
+    posRow.put("str_varchar", "abcd123456.");
+    posRow.put("bin", new byte[] {4, 5, 6});
+    posRow.put("bl", true);
+    posRow.put(
+        "var",
+        "{ \"a\": 1, \"b\": \"qwerty\", \"c\": null, \"d\": { \"e\": 2, \"f\": \"asdf\", \"g\":"
+            + " null } }");
+    posRow.put(
+        "obj",
+        "{ \"a\": 1, \"b\": \"qwerty\", \"c\": null, \"d\": { \"e\": 2, \"f\": \"asdf\", \"g\":"
+            + " null } }");
+    posRow.put("arr", Arrays.asList("{ \"a\": 1}", "{ \"b\": 2 }", "{ \"c\": 3 }"));
+    posRow.put("epochdays", "2022-09-18 20:05:07"); // DATE, 18.09.2022
+    posRow.put("epochsec", "2022-09-18 20:05:07"); // TIMESTAMP_NTZ(0)
+    posRow.put("epochnano", "2022-09-18 20:05:07.999999999"); // TIMESTAMP_NTZ(9)
+    posRow.put("timesec", "01:00:01.999999999"); // TIME(0)
+    posRow.put("timenano", "01:00:01.999999999"); // TIME(9)
+
+    return posRow;
   }
 
-  private static byte[] nextBytes(Random r) {
-    byte[] bin = new byte[128];
-    r.nextBytes(bin);
-    for (int i = 0; i < bin.length; i++) {
-      bin[i] = (byte) (Math.abs(bin[i]) % 25 + 97); // ascii letters
-    }
-    return bin;
+  private void assertNegRow(Map<String, Object> expectedNegRow, ResultSet actualResult)
+      throws SQLException {
+    assertNonTimeAndVarFields(expectedNegRow, actualResult);
+    Assert.assertEquals("{}", actualResult.getString("VAR"));
+    Assert.assertEquals("{}", actualResult.getString("OBJ"));
+    Assert.assertEquals("[]", actualResult.getString("ARR"));
+    Assert.assertEquals(0, actualResult.getDate("EPOCHDAYS").getTime());
+    Assert.assertEquals(0, actualResult.getTimestamp("EPOCHSEC").getTime());
+    Assert.assertEquals(0, actualResult.getTimestamp("EPOCHNANO").getNanos());
+    Assert.assertEquals(0, actualResult.getTimestamp("EPOCHNANO").getTime());
+    Assert.assertEquals(0, actualResult.getTimestamp("TIMESEC").getTime());
+    Assert.assertEquals(0, actualResult.getTimestamp("TIMENANO").getNanos());
+    Assert.assertEquals(0, actualResult.getTimestamp("TIMENANO").getTime());
+  }
+
+  private void assertNullRow(Map<String, Object> expectedNullRow, ResultSet actualResult)
+      throws SQLException {
+    Assert.assertNotNull(actualResult);
+    assertNonTimeAndVarFields(expectedNullRow, actualResult);
+    Assert.assertEquals(null, actualResult.getString("VAR"));
+    Assert.assertEquals(null, actualResult.getString("OBJ"));
+    Assert.assertEquals(null, actualResult.getString("ARR"));
+    Assert.assertEquals(null, actualResult.getDate("EPOCHDAYS"));
+    Assert.assertEquals(null, actualResult.getTimestamp("EPOCHSEC"));
+    Assert.assertEquals(null, actualResult.getTimestamp("EPOCHNANO"));
+    Assert.assertEquals(null, actualResult.getTimestamp("TIMESEC"));
+    Assert.assertEquals(null, actualResult.getTimestamp("TIMENANO"));
   }
 
-  private static double nextFloat(Random r) {
-    return (r.nextLong() % Math.round(Math.pow(10, 10))) / 100000d;
+  private void assertPosRow(Map<String, Object> expectedPosRow, ResultSet actualResult)
+      throws SQLException {
+    String formattedJSON =
+        "{\n"
+            + "  \"a\": 1,\n"
+            + "  \"b\": \"qwerty\",\n"
+            + "  \"c\": null,\n"
+            + "  \"d\": {\n"
+            + "    \"e\": 2,\n"
+            + "    \"f\": \"asdf\",\n"
+            + "    \"g\": null\n"
+            + "  }\n"
+            + "}";
+
+    String formattedArray =
+        "[\n"
+            + "  \"{ \\\"a\\\": 1}\",\n"
+            + "  \"{ \\\"b\\\": 2 }\",\n"
+            + "  \"{ \\\"c\\\": 3 }\"\n"
+            + "]";
+    Assert.assertNotNull(actualResult);
+    assertNonTimeAndVarFields(expectedPosRow, actualResult);
+    Assert.assertEquals(formattedJSON, actualResult.getString("VAR"));
+    Assert.assertEquals(formattedJSON, actualResult.getString("OBJ"));
+    Assert.assertEquals(formattedArray, actualResult.getString("ARR"));
+    Assert.assertEquals(
+        1663459200000l, actualResult.getDate("EPOCHDAYS").getTime()); // in ms, 18.09.2022 00:00:00
+    Assert.assertEquals(1663531507000L, actualResult.getTimestamp("EPOCHSEC").getTime());
+    Assert.assertEquals(999999999L, actualResult.getTimestamp("EPOCHNANO").getNanos());
+    // 1663531507000 ms + 999 ms (from ns)
+    Assert.assertEquals(1663531507999L, actualResult.getTimestamp("EPOCHNANO").getTime());
+    Assert.assertEquals(3601000, actualResult.getTimestamp("TIMESEC").getTime()); // 1h + 1s in ms
+    Assert.assertEquals(999999999, actualResult.getTimestamp("TIMENANO").getNanos());
+    // 1h + 1s + 999 ms (from ns)
+    Assert.assertEquals(3601999, actualResult.getTimestamp("TIMENANO").getTime());
   }
 
-  private static String nextJson(Random r) {
-    return String.format(
-        "{ \"%s\": %d, \"%s\": \"%s\", \"%s\": null, \"%s\": { \"%s\": %f, \"%s\": \"%s\", \"%s\":"
-            + " null } }",
-        nextString(r),
-        r.nextInt(),
-        nextString(r),
-        nextString(r),
-        nextString(r),
-        nextString(r),
-        nextString(r),
-        r.nextFloat(),
-        nextString(r),
-        nextString(r),
-        nextString(r));
+  private void assertNonTimeAndVarFields(Map<String, Object> row, ResultSet result)
+      throws SQLException {
+    Assert.assertNotNull(result);
+    Assert.assertEquals(row.get("num"), result.getInt("NUM"));
+    Assert.assertEquals(row.get("num_10_5"), result.getBigDecimal("NUM_10_5"));
+    Assert.assertEquals(row.get("num_38_0"), result.getBigDecimal("NUM_38_0"));
+    Assert.assertEquals(row.get("num_2_0"), result.getInt("NUM_2_0"));
+    Assert.assertEquals(row.get("num_4_0"), result.getInt("NUM_4_0"));
+    Assert.assertEquals(row.get("num_9_0"), result.getInt("NUM_9_0"));
+    Assert.assertEquals((long) row.get("num_18_0"), result.getLong("NUM_18_0"));
+    Assert.assertEquals((float) row.get("num_float"), result.getFloat("NUM_FLOAT"), 0);
+    Assert.assertEquals(row.get("str_varchar"), result.getString("STR_VARCHAR"));
+    Assert.assertArrayEquals((byte[]) row.get("bin"), result.getBytes("BIN"));
+    Assert.assertEquals(row.get("bl"), result.getBoolean("BL"));
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index 0ea0e5de4..31afda1f0 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -9,6 +9,8 @@
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
@@ -19,12 +21,27 @@
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.SFException;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
+@RunWith(Parameterized.class)
 public abstract class AbstractDataTypeTest {
+  @Parameterized.Parameters(name = "{0}")
+  public static Collection<Object[]> bdecVersion() {
+    return Arrays.asList(
+        new Object[][] {
+          {"Arrow", Constants.BdecVersion.ONE},
+          // TODO: uncomment once SNOW-659721 is deployed and we set the parameter
+          // DISABLE_PARQUET_CACHE to true for the test account
+          // {"Parquet", Constants.BdecVersion.THREE}
+        });
+  }
+
   private static final String SOURCE_COLUMN_NAME = "source";
   private static final String VALUE_COLUMN_NAME = "value";
 
@@ -52,6 +69,13 @@ protected String randomString() {
 
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
+  private final Constants.BdecVersion bdecVersion;
+
+  public AbstractDataTypeTest(
+      @SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
+    this.bdecVersion = bdecVersion;
+  }
+
   @Before
   public void before() throws Exception {
     databaseName = String.format("SDK_DATATYPE_COMPATIBILITY_IT_%s", randomString());
@@ -65,7 +89,13 @@ public void before() throws Exception {
 
     conn.createStatement().execute(String.format("use warehouse %s;", TestUtils.getWarehouse()));
 
-    Properties props = TestUtils.getProperties();
+    if (bdecVersion == Constants.BdecVersion.THREE) {
+      // TODO: encryption and interleaved mode are not yet supported by server side's Parquet
+      // scanner if local file cache is enabled (SNOW-656500)
+      conn.createStatement().execute("alter session set disable_parquet_cache=true;");
+    }
+
+    Properties props = TestUtils.getProperties(bdecVersion);
     if (props.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       props.setProperty(ROLE, "ACCOUNTADMIN");
     }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
index f0b0d8199..a7b9d6cd7 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
@@ -1,10 +1,15 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Ignore;
 import org.junit.Test;
 
 public class BinaryIT extends AbstractDataTypeTest {
 
+  public BinaryIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testBinarySimple() throws Exception {
     testJdbcTypeCompatibility("BINARY", new byte[0], new ByteArrayProvider());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
index bad5eac16..3a9445ea2 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
@@ -8,6 +8,7 @@
 import java.time.OffsetTime;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Ignore;
 import org.junit.Test;
 
@@ -17,6 +18,10 @@
  */
 public class DateTimeIT extends AbstractDataTypeTest {
 
+  public DateTimeIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testTimestampWithTimeZone() throws Exception {
     useLosAngelesTimeZone();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
index 1fdd2ad5e..48936b950 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
@@ -2,9 +2,15 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class LogicalTypesIT extends AbstractDataTypeTest {
+
+  public LogicalTypesIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testLogicalTypes() throws Exception {
     // Test booleans
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
index 57867427d..22c59d3ea 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
@@ -1,9 +1,15 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
 import java.util.Arrays;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class NullIT extends AbstractDataTypeTest {
+
+  public NullIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testNullIngestion() throws Exception {
     for (String type :
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
index 1b1c8d280..64c9bc3fc 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
@@ -2,9 +2,15 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class NumericTypesIT extends AbstractDataTypeTest {
+
+  public NumericTypesIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testIntegers() throws Exception {
     // test bytes
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
index 3c4ddde22..1195d2bfc 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
@@ -15,6 +15,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -24,6 +25,10 @@ public class SemiStructuredIT extends AbstractDataTypeTest {
   // server-side representation. Validation leaves a small buffer for this difference.
   private static final int MAX_ALLOWED_LENGTH = 16 * 1024 * 1024 - 64;
 
+  public SemiStructuredIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testVariant() throws Exception {
     // Test dates
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
index 4bcf39519..198d1c99c 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
@@ -2,10 +2,16 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Ignore;
 import org.junit.Test;
 
 public class StringsIT extends AbstractDataTypeTest {
+
+  public StringsIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testStrings() throws Exception {
     testJdbcTypeCompatibility("VARCHAR", "", new StringProvider());

From 83c6c0e383979202732f86fddbd320aacd2418ac Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 12 Oct 2022 15:39:00 -0700
Subject: [PATCH 078/356] NO-SNOW: add support to reset a rowIndex in the error
 response whenever needed (#241)

-add support to reset a rowIndex in the error response whenever needed
-update dependency version to fix Synk failure
---
 pom.xml                                              | 12 +-----------
 .../ingest/streaming/InsertValidationResponse.java   | 11 ++++++++++-
 .../SnowflakeStreamingIngestClientInternal.java      |  5 ++++-
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/pom.xml b/pom.xml
index e243039bd..ab7177278 100644
--- a/pom.xml
+++ b/pom.xml
@@ -211,21 +211,13 @@
             <version>9.9.3</version>
          </dependency>
 
-        <!-- Jackson for marshalling and unmarshalling JSON -->
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-core</artifactId>
-            <version>2.13.1</version>
-        </dependency>
-
         <!-- Jackson Databind api -->
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.13.4</version>
+            <version>2.14.0-rc1</version>
         </dependency>
 
-
         <!-- SLF4J api that a client can shim in later -->
         <dependency>
             <groupId>org.slf4j</groupId>
@@ -234,7 +226,6 @@
             <scope>provided</scope>
         </dependency>
 
-
         <!-- JDK logger backend for logging tests -->
         <dependency>
             <groupId>org.slf4j</groupId>
@@ -252,7 +243,6 @@
             <version>2.3.1</version>
         </dependency>
 
-
         <!-- JUnit so that we can make some basic unit tests -->
         <dependency>
             <groupId>junit</groupId>
diff --git a/src/main/java/net/snowflake/ingest/streaming/InsertValidationResponse.java b/src/main/java/net/snowflake/ingest/streaming/InsertValidationResponse.java
index 0a1deae28..f57d384c9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/InsertValidationResponse.java
+++ b/src/main/java/net/snowflake/ingest/streaming/InsertValidationResponse.java
@@ -52,7 +52,7 @@ public static class InsertError {
     // Used to map this error row with original row in the insertRows Iterable.
     // i.e the rowIndex can be index 9 in the list of 10 rows.
     // index is 0 based so as to match with incoming Iterable
-    private final long rowIndex;
+    private long rowIndex;
 
     // List of extra column names in the input row compared with the table schema
     private List<String> extraColNames;
@@ -89,6 +89,15 @@ public SFException getException() {
       return this.exception;
     }
 
+    /**
+     * Set the row index
+     *
+     * @param rowIndex the corresponding row index in the original input row list
+     */
+    public void setRowIndex(long rowIndex) {
+      this.rowIndex = rowIndex;
+    }
+
     /**
      * Get the rowIndex. Please note, this index is 0 based so it can be used in fetching nth row
      * from the input. ({@link SnowflakeStreamingIngestChannel#insertRows(Iterable, String)})
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 169be2403..8be3c4ed3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -304,9 +304,12 @@ public SnowflakeStreamingIngestChannelInternal<?> openChannel(OpenChannelRequest
       }
 
       logger.logInfo(
-          "Open channel request succeeded, channel={}, table={}, client={}",
+          "Open channel request succeeded, channel={}, table={}, clientSequencer={},"
+              + " rowSequencer={}, client={}",
           request.getChannelName(),
           request.getFullyQualifiedTableName(),
+          response.getClientSequencer(),
+          response.getRowSequencer(),
           getName());
 
       // Channel is now registered, add it to the in-memory channel pool

From f6e202a00040cf653da3a69d843a11a3102aed38 Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Fri, 14 Oct 2022 09:46:16 +0200
Subject: [PATCH 079/356] Revert "@snow SNOW-633432 Parquet file generator
 (#200)" (#249)

This reverts commit 7222174563a9a1941a2559feb05f638da8130321.
---
 pom.xml                                       |  60 --
 scripts/check_content.sh                      |  18 +-
 .../streaming/internal/ArrowRowBuffer.java    |  10 +-
 .../streaming/internal/BlobMetadata.java      |  17 +-
 .../internal/BlobMetadataWithBdecVersion.java |  25 -
 .../streaming/internal/ChannelCache.java      |   3 +-
 .../internal/DataValidationUtil.java          |  11 -
 .../streaming/internal/FlushService.java      |   7 +-
 .../ingest/streaming/internal/Flusher.java    |   3 +-
 .../streaming/internal/ParquetChunkData.java  |  25 -
 .../streaming/internal/ParquetFlusher.java    | 348 -----------
 .../streaming/internal/ParquetRowBuffer.java  | 222 -------
 .../internal/ParquetTypeGenerator.java        | 292 ---------
 .../internal/ParquetValueParser.java          | 323 ----------
 .../streaming/internal/RegisterService.java   |   3 +-
 .../ingest/streaming/internal/RowBuffer.java  |   3 +-
 ...owflakeStreamingIngestChannelInternal.java |  20 +-
 ...nowflakeStreamingIngestClientInternal.java |   5 +-
 .../net/snowflake/ingest/utils/Constants.java |   9 +-
 .../java/net/snowflake/ingest/TestUtils.java  |  12 +-
 .../streaming/internal/ArrowBufferTest.java   | 242 ++++----
 .../internal/ColumnMetadataBuilder.java       | 164 -----
 .../streaming/internal/FlushServiceTest.java  |  50 +-
 .../internal/ParquetTypeGeneratorTest.java    | 480 ---------------
 .../internal/ParquetValueParserTest.java      | 570 ------------------
 .../streaming/internal/RowBufferTest.java     |   7 +-
 .../streaming/internal/StreamingIngestIT.java | 289 +++------
 .../datatypes/AbstractDataTypeTest.java       |  32 +-
 .../internal/datatypes/BinaryIT.java          |   5 -
 .../internal/datatypes/DateTimeIT.java        |   5 -
 .../internal/datatypes/LogicalTypesIT.java    |   6 -
 .../streaming/internal/datatypes/NullIT.java  |   6 -
 .../internal/datatypes/NumericTypesIT.java    |   6 -
 .../internal/datatypes/SemiStructuredIT.java  |   5 -
 .../internal/datatypes/StringsIT.java         |   6 -
 35 files changed, 262 insertions(+), 3027 deletions(-)
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
 delete mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
 delete mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
 delete mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java

diff --git a/pom.xml b/pom.xml
index ab7177278..cb6b2eda0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -43,7 +43,6 @@
         <jacoco.skip.instrument>true</jacoco.skip.instrument>
         <jacoco.version>0.8.5</jacoco.version>
         <arrow.version>9.0.0</arrow.version>
-        <hadoop.version>2.10.1</hadoop.version>
     </properties>
 
     <build>
@@ -293,65 +292,6 @@
             <version>${arrow.version}</version>
         </dependency>
 
-        <dependency>
-            <groupId>org.apache.parquet</groupId>
-            <artifactId>parquet-hadoop</artifactId>
-            <version>1.11.2</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>javax.annotation</groupId>
-                    <artifactId>javax.annotation-api</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-common</artifactId>
-            <version>${hadoop.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>com.github.spotbugs</groupId>
-                    <artifactId>spotbugs-annotations</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.sun.jersey</groupId>
-                    <artifactId>jersey-core</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.sun.jersey</groupId>
-                    <artifactId>jersey-json</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.sun.jersey</groupId>
-                    <artifactId>jersey-server</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>javax.servlet</groupId>
-                    <artifactId>servlet-api</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>javax.servlet.jsp</groupId>
-                    <artifactId>jsp-api</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>log4j</groupId>
-                    <artifactId>log4j</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.codehaus.jackson</groupId>
-                    <artifactId>jackson-mapper-asl</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.google.protobuf</groupId>
-                    <artifactId>protobuf-java</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.mortbay.jetty</groupId>
-                    <artifactId>jetty</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-
         <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-core -->
         <dependency>
             <groupId>io.dropwizard.metrics</groupId>
diff --git a/scripts/check_content.sh b/scripts/check_content.sh
index d2594d583..9e5b1c404 100755
--- a/scripts/check_content.sh
+++ b/scripts/check_content.sh
@@ -5,23 +5,7 @@
 set -o pipefail
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-if jar tvf $DIR/../target/snowflake-ingest-sdk.jar  | awk '{print $8}' | grep -v -E "^(net|com)/snowflake" \
-    | grep -v -E "(com|net)/\$" | grep -v -E "^META-INF" | grep -v -E "^mozilla" | grep -v mime.types \
-    | grep -v project.properties | grep -v -E "javax" | grep -v -E "^org/" | grep -v -E "^com/google" \
-    | grep -v -E "^com/sun" | grep -v "log4j.properties" | grep -v "git.properties" | grep -v "io/" \
-    | grep -v "codegen/" | grep -v "com/codahale/" | grep -v "com/ibm/" | grep -v "LICENSE" | grep -v "aix/" \
-    | grep -v "darwin/" | grep -v "win/" | grep -v "freebsd/" | grep -v "linux/" | grep -v "com/github/" \
-    | grep -v -E "shaded/" | grep -v "webapps/"  | grep -v "microsoft/" | grep -v "com/ctc/" \
-    | grep -v "edu/" | grep -v "com/jcraft/" | grep -v "contribs/" | grep -v "com/zaxxer/"  | grep -v "com/squareup/" \
-    | grep -v "com/thoughtworks/" | grep -v "com/jamesmurty/"  | grep -v "net/iharder/" \
-    | grep -v -E "^core-default.xml" \
-    | grep -v "yarn-version-info.properties" | grep -v "yarn-version-info.properties" \
-    | grep -v "common-version-info.properties" | grep -v "LocalizedFormats_fr.properties" \
-    | grep -v "org.apache.hadoop.application-classloader.properties" | grep -v "assets/" | grep -v "ehcache-core.xsd" \
-    | grep -v "ehcache-107ext.xsd" | grep -v "parquet.thrift" | grep -v "mapred-default.xml" \
-    | grep -v "yarn-default.xml" | grep -v ".keep"  | grep -v "NOTICE" | grep -v "digesterRules.xml" \
-    | grep -v "properties.dtd" | grep -v "PropertyList-1.0.dtd"  \
-    ; then
+if jar tvf $DIR/../target/snowflake-ingest-sdk.jar  | awk '{print $8}' | grep -v -E "^(net|com)/snowflake" | grep -v -E "(com|net)/\$" | grep -v -E "^META-INF" | grep -v -E "^mozilla" | grep -v mime.types | grep -v project.properties | grep -v -E "javax" | grep -v -E "^org/" | grep -v -E "^com/google" | grep -v -E "^com/sun" | grep -v "log4j.properties" | grep -v "git.properties" | grep -v "io/" | grep -v "codegen/" | grep -v "com/codahale/" | grep -v "com/ibm/" | grep -v "LICENSE" | grep -v "aix/" | grep -v "darwin/" | grep -v "win/" | grep -v "freebsd/" | grep -v "linux/" | grep -v "com/github/"; then
   echo "[ERROR] Ingest SDK jar includes class not under the snowflake namespace"
   exit 1
 fi
\ No newline at end of file
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 2d7d17a92..38b4583a9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -471,7 +471,15 @@ private float convertRowToArrow(
             // vector.setSafe requires the BigDecimal input scale explicitly match its scale
             inputAsBigDecimal = inputAsBigDecimal.setScale(columnScale, RoundingMode.HALF_UP);
 
-            DataValidationUtil.checkValueInRange(inputAsBigDecimal, columnScale, columnPrecision);
+            if (inputAsBigDecimal.abs().compareTo(BigDecimal.TEN.pow(columnPrecision - columnScale))
+                >= 0) {
+              throw new SFException(
+                  ErrorCode.INVALID_ROW,
+                  inputAsBigDecimal,
+                  String.format(
+                      "Number out of representable exclusive range of (-1e%s..1e%s)",
+                      columnPrecision - columnScale, columnPrecision - columnScale));
+            }
 
             if (columnScale != 0 || physicalType == ColumnPhysicalType.SB16) {
               ((DecimalVector) vector).setSafe(curRowIndex, inputAsBigDecimal);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index f90ba5055..2818a392a 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -49,16 +49,9 @@ List<ChunkMetadata> getChunks() {
     return this.chunks;
   }
 
-  /**
-   * Create {@link BlobMetadata} in case of {@link Constants.BdecVersion#ONE} and {@link
-   * BlobMetadataWithBdecVersion} otherwise to send BDEC version to server side.
-   */
-  static BlobMetadata createBlobMetadata(
-      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks) {
-    // TODO SNOW-659721: Unify BlobMetadata with BlobMetadataWithBdecVersion once server side
-    // BdecVersion in production
-    return bdecVersion == Constants.BdecVersion.ONE
-        ? new BlobMetadata(path, md5, bdecVersion, chunks)
-        : new BlobMetadataWithBdecVersion(path, md5, bdecVersion, chunks);
-  }
+  // TODO: send the bdec_version once server side supports this in production
+  //  @JsonProperty("bdec_version")
+  //  byte getVersionByte() {
+  //    return bdecVersion.toByte();
+  //  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
deleted file mode 100644
index 7f9c602e3..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import java.util.List;
-import net.snowflake.ingest.utils.Constants;
-
-/**
- * Same as {@link BlobMetadata} but with {@link Constants.BdecVersion} to send it to server side if
- * the version is greater than {@link Constants.BdecVersion#ONE}.
- */
-class BlobMetadataWithBdecVersion extends BlobMetadata {
-  BlobMetadataWithBdecVersion(
-      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks) {
-    super(path, md5, bdecVersion, chunks);
-  }
-
-  @JsonProperty("bdec_version")
-  byte getVersionByte() {
-    return getVersion().toByte();
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
index ef366b120..35ca580bc 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
@@ -14,8 +14,7 @@
  * during flush. The key is a fully qualified table name and the value is a set of channels that
  * belongs to this table
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
 class ChannelCache<T> {
   // Cache to hold all the valid channels, the key for the outer map is FullyQualifiedTableName and
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index de7185009..881a4c2f3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -674,17 +674,6 @@ static int validateAndParseBoolean(Object input) {
         input.getClass(), "BOOLEAN", new String[] {"boolean", "Number", "String"});
   }
 
-  static void checkValueInRange(BigDecimal bigDecimalValue, int scale, int precision) {
-    if (bigDecimalValue.abs().compareTo(BigDecimal.TEN.pow(precision - scale)) >= 0) {
-      throw new SFException(
-          ErrorCode.INVALID_ROW,
-          bigDecimalValue,
-          String.format(
-              "Number out of representable exclusive range of (-1e%s..1e%s)",
-              precision - scale, precision - scale));
-    }
-  }
-
   static Set<String> allowedBooleanStringsLowerCased =
       Sets.newHashSet("1", "0", "yes", "no", "y", "n", "t", "f", "true", "false", "on", "off");
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 0ef790c6f..df5522469 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -58,8 +58,7 @@
  * <li>upload the blob to stage
  * <li>register the blob to the targeted Snowflake table
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
 class FlushService<T> {
 
@@ -468,6 +467,7 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
         // We need to maintain IV as a block counter for the whole file, even interleaved,
         // to align with decryption on the Snowflake query path.
         // TODO: address alignment for the header SNOW-557866
+        // TODO: encryption is not yet supported by server side for Parquet
         long iv = curDataSize / Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES;
         byte[] encryptedCompressedChunkData =
             Cryptor.encrypt(
@@ -559,8 +559,7 @@ BlobMetadata upload(String filePath, byte[] blob, List<ChunkMetadata> metadata)
         blob.length,
         System.currentTimeMillis() - startTime);
 
-    return BlobMetadata.createBlobMetadata(
-        filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata);
+    return new BlobMetadata(filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
index 37ab4afc4..4ac7acf8c 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
@@ -13,8 +13,7 @@
  * Interface to convert {@link ChannelData} buffered in {@link RowBuffer} to the underlying format
  * implementation for faster processing.
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
 public interface Flusher<T> {
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
deleted file mode 100644
index 0c7b12ff2..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-import java.util.List;
-import java.util.Map;
-
-/** Parquet data holder to buffer rows. */
-public class ParquetChunkData {
-  final List<List<Object>> rows;
-  final Map<String, String> metadata;
-
-  /**
-   * Construct parquet data chunk.
-   *
-   * @param rows chunk row set
-   * @param metadata chunk metadata
-   */
-  public ParquetChunkData(List<List<Object>> rows, Map<String, String> metadata) {
-    this.rows = rows;
-    this.metadata = metadata;
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
deleted file mode 100644
index 3d9827b6b..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-import static net.snowflake.ingest.utils.Constants.MAX_CHUNK_SIZE_IN_BYTES;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.SFException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.parquet.column.ColumnDescriptor;
-import org.apache.parquet.column.ParquetProperties;
-import org.apache.parquet.hadoop.ParquetFileWriter;
-import org.apache.parquet.hadoop.ParquetWriter;
-import org.apache.parquet.hadoop.api.WriteSupport;
-import org.apache.parquet.hadoop.metadata.CompressionCodecName;
-import org.apache.parquet.io.DelegatingPositionOutputStream;
-import org.apache.parquet.io.OutputFile;
-import org.apache.parquet.io.ParquetEncodingException;
-import org.apache.parquet.io.PositionOutputStream;
-import org.apache.parquet.io.api.Binary;
-import org.apache.parquet.io.api.RecordConsumer;
-import org.apache.parquet.schema.MessageType;
-import org.apache.parquet.schema.PrimitiveType;
-
-/**
- * Converts {@link ChannelData} buffered in {@link RowBuffer} to the Parquet format for faster
- * processing.
- */
-public class ParquetFlusher implements Flusher<ParquetChunkData> {
-  private static final Logging logger = new Logging(ParquetFlusher.class);
-  private final MessageType schema;
-
-  /** Construct parquet flusher from its schema. */
-  public ParquetFlusher(MessageType schema) {
-    this.schema = schema;
-  }
-
-  @Override
-  public SerializationResult serialize(
-      List<ChannelData<ParquetChunkData>> channelsDataPerTable,
-      ByteArrayOutputStream chunkData,
-      String filePath)
-      throws IOException {
-    List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
-    long rowCount = 0L;
-    SnowflakeStreamingIngestChannelInternal<ParquetChunkData> firstChannel = null;
-    Map<String, RowBufferStats> columnEpStatsMapCombined = null;
-    List<List<Object>> rows = null;
-
-    for (ChannelData<ParquetChunkData> data : channelsDataPerTable) {
-      // Create channel metadata
-      ChannelMetadata channelMetadata =
-          ChannelMetadata.builder()
-              .setOwningChannel(data.getChannel())
-              .setRowSequencer(data.getRowSequencer())
-              .setOffsetToken(data.getOffsetToken())
-              .build();
-      // Add channel metadata to the metadata list
-      channelsMetadataList.add(channelMetadata);
-
-      logger.logDebug(
-          "Parquet Flusher: Start building channel={}, rowCount={}, bufferSize={} in blob={}",
-          data.getChannel().getFullyQualifiedName(),
-          data.getRowCount(),
-          data.getBufferSize(),
-          filePath);
-
-      if (rows == null) {
-        columnEpStatsMapCombined = data.getColumnEps();
-        rows = new ArrayList<>();
-        firstChannel = data.getChannel();
-      } else {
-        // This method assumes that channelsDataPerTable is grouped by table. We double check
-        // here and throw an error if the assumption is violated
-        if (!data.getChannel()
-            .getFullyQualifiedTableName()
-            .equals(firstChannel.getFullyQualifiedTableName())) {
-          throw new SFException(ErrorCode.INVALID_DATA_IN_CHUNK);
-        }
-
-        columnEpStatsMapCombined =
-            ChannelData.getCombinedColumnStatsMap(columnEpStatsMapCombined, data.getColumnEps());
-      }
-      rows.addAll(data.getVectors().rows);
-
-      rowCount += data.getRowCount();
-
-      logger.logDebug(
-          "Parquet Flusher: Finish building channel={}, rowCount={}, bufferSize={} in blob={}",
-          data.getChannel().getFullyQualifiedName(),
-          data.getRowCount(),
-          data.getBufferSize(),
-          filePath);
-    }
-
-    Map<String, String> metadata = channelsDataPerTable.get(0).getVectors().metadata;
-
-    flushToParquetBdecChunk(chunkData, rows, metadata, channelsMetadataList);
-    return new SerializationResult(channelsMetadataList, columnEpStatsMapCombined, rowCount);
-  }
-
-  /**
-   * Flushes a parquet row chunk to the given BDEC output stream.
-   *
-   * @param bdecOutput BDEC output stream
-   * @param chunkRows chunk rows
-   * @param metadata chunk metadata
-   * @param channelsMetadataList metadata of the channels the chunk rows belong to
-   * @throws IOException thrown from Parquet library in case of writing problems
-   */
-  private void flushToParquetBdecChunk(
-      ByteArrayOutputStream bdecOutput,
-      List<List<Object>> chunkRows,
-      Map<String, String> metadata,
-      List<ChannelMetadata> channelsMetadataList)
-      throws IOException {
-    try {
-      ParquetWriter<List<Object>> writer =
-          new BdecParquetWriterBuilder(bdecOutput, schema, metadata, channelsMetadataList)
-              // PARQUET_2_0 uses Encoding.DELTA_BYTE_ARRAY for byte arrays (e.g. SF sb16)
-              // server side does not support it TODO: SNOW-657238
-              .withWriterVersion(ParquetProperties.WriterVersion.PARQUET_1_0)
-
-              // the dictionary encoding (Encoding.*_DICTIONARY) is not supported by server side
-              // scanner yet
-              .withDictionaryEncoding(false)
-
-              // Historically server side scanner supports only the case when the row number in all
-              // pages is the same.
-              // The quick fix is to effectively disable the page size/row limit
-              // to always have one page per chunk until server side is generalised.
-              .withPageSize((int) Constants.MAX_CHUNK_SIZE_IN_BYTES * 2)
-              .withPageRowCountLimit(chunkRows.size() + 1)
-              .enableValidation()
-              .withCompressionCodec(CompressionCodecName.GZIP)
-              .withWriteMode(ParquetFileWriter.Mode.CREATE)
-              .build();
-
-      // We can use lower level column writers and custom ValuesWriterFactory that uses plain byte
-      // array encoding used by PARQUET_1_0 and supported by server side
-      // TODO: SNOW-672143
-      for (List<Object> row : chunkRows) {
-        writer.write(row);
-      }
-      writer.close();
-    } catch (Throwable t) {
-      logger.logError("Parquet Flusher: failed to write", t);
-      throw t;
-    }
-  }
-
-  /**
-   * A parquet specific write builder.
-   *
-   * <p>This class is implemented as parquet library API requires, mostly to provide {@link
-   * BdecWriteSupport} implementation.
-   */
-  private static class BdecParquetWriterBuilder
-      extends ParquetWriter.Builder<List<Object>, BdecParquetWriterBuilder> {
-    private final MessageType schema;
-    private final Map<String, String> extraMetaData;
-    private final List<ChannelMetadata> channelsMetadataList;
-
-    protected BdecParquetWriterBuilder(
-        ByteArrayOutputStream stream,
-        MessageType schema,
-        Map<String, String> extraMetaData,
-        List<ChannelMetadata> channelsMetadataList) {
-      super(new ByteArrayOutputFile(stream));
-      this.schema = schema;
-      this.extraMetaData = extraMetaData;
-      this.channelsMetadataList = channelsMetadataList;
-    }
-
-    @Override
-    protected BdecParquetWriterBuilder self() {
-      return this;
-    }
-
-    @Override
-    protected WriteSupport<List<Object>> getWriteSupport(Configuration conf) {
-      return new BdecWriteSupport(schema, extraMetaData, channelsMetadataList);
-    }
-  }
-
-  /**
-   * A parquet specific file output implementation.
-   *
-   * <p>This class is implemented as parquet library API requires, mostly to create our {@link
-   * ByteArrayDelegatingPositionOutputStream} implementation.
-   */
-  private static class ByteArrayOutputFile implements OutputFile {
-    private final ByteArrayOutputStream stream;
-
-    private ByteArrayOutputFile(ByteArrayOutputStream stream) {
-      this.stream = stream;
-    }
-
-    @Override
-    public PositionOutputStream create(long blockSizeHint) throws IOException {
-      stream.reset();
-      return new ByteArrayDelegatingPositionOutputStream(stream);
-    }
-
-    @Override
-    public PositionOutputStream createOrOverwrite(long blockSizeHint) throws IOException {
-      return create(blockSizeHint);
-    }
-
-    @Override
-    public boolean supportsBlockSize() {
-      return false;
-    }
-
-    @Override
-    public long defaultBlockSize() {
-      return (int) MAX_CHUNK_SIZE_IN_BYTES;
-    }
-  }
-
-  /**
-   * A parquet specific output stream implementation.
-   *
-   * <p>This class is implemented as parquet library API requires, mostly to wrap our BDEC output
-   * {@link ByteArrayOutputStream}.
-   */
-  private static class ByteArrayDelegatingPositionOutputStream
-      extends DelegatingPositionOutputStream {
-    private final ByteArrayOutputStream stream;
-
-    public ByteArrayDelegatingPositionOutputStream(ByteArrayOutputStream stream) {
-      super(stream);
-      this.stream = stream;
-    }
-
-    @Override
-    public long getPos() {
-      return stream.size();
-    }
-  }
-
-  /**
-   * A parquet specific write support implementation.
-   *
-   * <p>This class is implemented as parquet library API requires, mostly to serialize user column
-   * values depending on type into Parquet {@link RecordConsumer} in {@link
-   * BdecWriteSupport#write(List)}.
-   */
-  private static class BdecWriteSupport extends WriteSupport<List<Object>> {
-    MessageType schema;
-    RecordConsumer recordConsumer;
-    Map<String, String> extraMetadata;
-    List<ChannelMetadata> channelsMetadataList;
-
-    // TODO SNOW-672156: support specifying encodings and compression
-    BdecWriteSupport(
-        MessageType schema,
-        Map<String, String> extraMetadata,
-        List<ChannelMetadata> channelsMetadataList) {
-      this.schema = schema;
-      this.extraMetadata = extraMetadata;
-      this.channelsMetadataList = channelsMetadataList;
-    }
-
-    @Override
-    public WriteContext init(Configuration config) {
-      return new WriteContext(schema, extraMetadata);
-    }
-
-    @Override
-    public void prepareForWrite(RecordConsumer recordConsumer) {
-      this.recordConsumer = recordConsumer;
-    }
-
-    @Override
-    public void write(List<Object> values) {
-      List<ColumnDescriptor> cols = schema.getColumns();
-      if (values.size() != cols.size()) {
-        List<String> channelNames =
-            this.channelsMetadataList.stream()
-                .map(ChannelMetadata::getChannelName)
-                .collect(Collectors.toList());
-        throw new ParquetEncodingException(
-            "Invalid input data in channels "
-                + channelNames
-                + ". Expecting "
-                + cols.size()
-                + " columns. Input had "
-                + values.size()
-                + " columns ("
-                + cols
-                + ") : "
-                + values);
-      }
-
-      recordConsumer.startMessage();
-      for (int i = 0; i < cols.size(); ++i) {
-        Object val = values.get(i);
-        // val.length() == 0 indicates a NULL value.
-        if (val != null) {
-          String fieldName = cols.get(i).getPath()[0];
-          recordConsumer.startField(fieldName, i);
-          PrimitiveType.PrimitiveTypeName typeName =
-              cols.get(i).getPrimitiveType().getPrimitiveTypeName();
-          switch (typeName) {
-            case BOOLEAN:
-              recordConsumer.addBoolean((boolean) val);
-              break;
-            case FLOAT:
-              recordConsumer.addFloat((float) val);
-              break;
-            case DOUBLE:
-              recordConsumer.addDouble((double) val);
-              break;
-            case INT32:
-              recordConsumer.addInteger((int) val);
-              break;
-            case INT64:
-              recordConsumer.addLong((long) val);
-              break;
-            case BINARY:
-              recordConsumer.addBinary(Binary.fromString((String) val));
-              break;
-            case FIXED_LEN_BYTE_ARRAY:
-              Binary binary = Binary.fromConstantByteArray((byte[]) val);
-              recordConsumer.addBinary(binary);
-              break;
-            default:
-              throw new ParquetEncodingException(
-                  "Unsupported column type: " + cols.get(i).getPrimitiveType());
-          }
-          recordConsumer.endField(fieldName, i);
-        }
-      }
-      recordConsumer.endMessage();
-    }
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
deleted file mode 100644
index de4b3eadc..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import org.apache.parquet.column.ColumnDescriptor;
-import org.apache.parquet.schema.MessageType;
-import org.apache.parquet.schema.PrimitiveType;
-import org.apache.parquet.schema.Type;
-
-/**
- * The buffer in the Streaming Ingest channel that holds the un-flushed rows, these rows will be
- * converted to Parquet format for faster processing
- */
-public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
-  private static final Logging logger = new Logging(ParquetRowBuffer.class);
-
-  private static final String PARQUET_MESSAGE_TYPE_NAME = "bdec";
-
-  private final Map<String, Pair<ColumnMetadata, Integer>> fieldIndex;
-  private final Map<String, String> metadata;
-  private final List<List<Object>> data;
-  private final List<List<Object>> tempData;
-
-  private MessageType schema;
-
-  /**
-   * Construct a ParquetRowBuffer object
-   *
-   * @param channel client channel
-   */
-  ParquetRowBuffer(SnowflakeStreamingIngestChannelInternal<ParquetChunkData> channel) {
-    super(channel);
-    fieldIndex = new HashMap<>();
-    metadata = new HashMap<>();
-    data = new ArrayList<>();
-    tempData = new ArrayList<>();
-  }
-
-  @Override
-  public void setupSchema(List<ColumnMetadata> columns) {
-    fieldIndex.clear();
-    metadata.clear();
-    List<Type> parquetTypes = new ArrayList<>();
-    // Snowflake column id that corresponds to the order in 'columns' received from server
-    // id is required to pack column metadata for the server scanner, e.g. decimal scale and
-    // precision
-    int id = 1;
-    for (ColumnMetadata column : columns) {
-      ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-          ParquetTypeGenerator.generateColumnParquetTypeInfo(column, id);
-      parquetTypes.add(typeInfo.getParquetType());
-      this.metadata.putAll(typeInfo.getMetadata());
-      fieldIndex.put(column.getName(), new Pair<>(column, parquetTypes.size() - 1));
-      if (!column.getNullable()) {
-        addNonNullableFieldName(column.getName());
-      }
-      this.statsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
-
-      if (this.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.ABORT) {
-        this.tempStatsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
-      }
-
-      id++;
-    }
-    schema = new MessageType(PARQUET_MESSAGE_TYPE_NAME, parquetTypes);
-  }
-
-  @Override
-  boolean hasColumn(String name) {
-    return fieldIndex.containsKey(name);
-  }
-
-  @Override
-  float addRow(
-      Map<String, Object> row,
-      int curRowIndex,
-      Map<String, RowBufferStats> statsMap,
-      Set<String> formattedInputColumnNames) {
-    return addRow(row, data, statsMap, formattedInputColumnNames);
-  }
-
-  @Override
-  float addTempRow(
-      Map<String, Object> row,
-      int curRowIndex,
-      Map<String, RowBufferStats> statsMap,
-      Set<String> formattedInputColumnNames) {
-    return addRow(row, tempData, statsMap, formattedInputColumnNames);
-  }
-
-  /**
-   * Adds a row to the parquet buffer.
-   *
-   * @param row row to add
-   * @param out internal buffer to add to
-   * @param statsMap column stats map
-   * @param inputColumnNames list of input column names after formatting
-   * @return row size
-   */
-  private float addRow(
-      Map<String, Object> row,
-      List<List<Object>> out,
-      Map<String, RowBufferStats> statsMap,
-      Set<String> inputColumnNames) {
-    Object[] indexedRow = new Object[fieldIndex.size()];
-    float size = 0F;
-    for (Map.Entry<String, Object> entry : row.entrySet()) {
-      String key = entry.getKey();
-      Object value = entry.getValue();
-      String columnName = formatColumnName(key);
-      int colIndex = fieldIndex.get(columnName).getSecond();
-      RowBufferStats stats = statsMap.get(columnName);
-      ColumnMetadata column = fieldIndex.get(columnName).getFirst();
-      ColumnDescriptor columnDescriptor = schema.getColumns().get(colIndex);
-      PrimitiveType.PrimitiveTypeName typeName =
-          columnDescriptor.getPrimitiveType().getPrimitiveTypeName();
-      ParquetValueParser.ParquetBufferValue valueWithSize =
-          ParquetValueParser.parseColumnValueToParquet(value, column, typeName, stats);
-      indexedRow[colIndex] = valueWithSize.getValue();
-      size += valueWithSize.getSize();
-    }
-    out.add(Arrays.asList(indexedRow));
-
-    for (String columnName : Sets.difference(this.fieldIndex.keySet(), inputColumnNames)) {
-      statsMap.get(columnName).incCurrentNullCount();
-    }
-    return size;
-  }
-
-  @Override
-  void moveTempRowsToActualBuffer(int tempRowCount) {
-    data.addAll(tempData);
-  }
-
-  @Override
-  void clearTempRows() {
-    tempData.clear();
-  }
-
-  @Override
-  boolean hasColumns() {
-    return !fieldIndex.isEmpty();
-  }
-
-  @Override
-  Optional<ParquetChunkData> getSnapshot() {
-    List<List<Object>> oldData = new ArrayList<>();
-    data.forEach(r -> oldData.add(new ArrayList<>(r)));
-    return oldData.isEmpty()
-        ? Optional.empty()
-        : Optional.of(new ParquetChunkData(oldData, metadata));
-  }
-
-  /** Used only for testing. */
-  @Override
-  Object getVectorValueAt(String column, int index) {
-    int colIndex = fieldIndex.get(column).getSecond();
-    Object value = data.get(index).get(colIndex);
-    ColumnMetadata columnMetadata = fieldIndex.get(column).getFirst();
-    String physicalTypeStr = columnMetadata.getPhysicalType();
-    ColumnPhysicalType physicalType = ColumnPhysicalType.valueOf(physicalTypeStr);
-    String logicalTypeStr = columnMetadata.getLogicalType();
-    ColumnLogicalType logicalType = ColumnLogicalType.valueOf(logicalTypeStr);
-    if (logicalType == ColumnLogicalType.FIXED) {
-      if (physicalType == ColumnPhysicalType.SB1) {
-        value = ((Integer) value).byteValue();
-      }
-      if (physicalType == ColumnPhysicalType.SB2) {
-        value = ((Integer) value).shortValue();
-      }
-      if (physicalType == ColumnPhysicalType.SB16) {
-        value = new BigDecimal(new BigInteger((byte[]) value), columnMetadata.getScale());
-      }
-    }
-    if (logicalType == ColumnLogicalType.BINARY && value != null) {
-      value = ((String) value).getBytes(StandardCharsets.UTF_8);
-    }
-    return value;
-  }
-
-  @Override
-  int getTempRowCount() {
-    return tempData.size();
-  }
-
-  @Override
-  void reset() {
-    super.reset();
-    data.clear();
-  }
-
-  @Override
-  public void close(String name) {
-    this.fieldIndex.clear();
-    logger.logInfo(
-        "Trying to close parquet buffer for channel={} from function={}",
-        this.owningChannel.getName(),
-        name);
-  }
-
-  @Override
-  public Flusher<ParquetChunkData> createFlusher(Constants.BdecVersion bdecVerion) {
-    return new ParquetFlusher(schema);
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
deleted file mode 100644
index 149ce766c..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/*
- * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.SFException;
-import org.apache.parquet.schema.LogicalTypeAnnotation;
-import org.apache.parquet.schema.PrimitiveType;
-import org.apache.parquet.schema.Type;
-import org.apache.parquet.schema.Types;
-
-/** Generates the Parquet types for the Snowflake's column types */
-public class ParquetTypeGenerator {
-
-  /**
-   * Util class that contains Parquet type and other metadata for that type needed by the Snowflake
-   * server side scanner
-   */
-  static class ParquetTypeInfo {
-    private Type parquetType;
-    private Map<String, String> metadata;
-
-    public Type getParquetType() {
-      return this.parquetType;
-    }
-
-    public Map<String, String> getMetadata() {
-      return this.metadata;
-    }
-
-    public void setParquetType(Type parquetType) {
-      this.parquetType = parquetType;
-    }
-
-    public void setMetadata(Map<String, String> metadata) {
-      this.metadata = metadata;
-    }
-  }
-
-  private static final Set<AbstractRowBuffer.ColumnPhysicalType> TIME_SUPPORTED_PHYSICAL_TYPES =
-      new HashSet<>(
-          Arrays.asList(
-              AbstractRowBuffer.ColumnPhysicalType.SB4, AbstractRowBuffer.ColumnPhysicalType.SB8));
-  private static final Set<AbstractRowBuffer.ColumnPhysicalType>
-      TIMESTAMP_SUPPORTED_PHYSICAL_TYPES =
-          new HashSet<>(
-              Arrays.asList(
-                  AbstractRowBuffer.ColumnPhysicalType.SB8,
-                  AbstractRowBuffer.ColumnPhysicalType.SB16));
-
-  /**
-   * Generate the column parquet type and metadata from the column metadata received from server
-   * side.
-   *
-   * @param column column metadata as received from server side
-   * @param id column id
-   * @return column parquet type
-   */
-  static ParquetTypeInfo generateColumnParquetTypeInfo(ColumnMetadata column, int id) {
-    ParquetTypeInfo res = new ParquetTypeInfo();
-    Type parquetType;
-    Map<String, String> metadata = new HashMap<>();
-    String name = column.getName();
-
-    AbstractRowBuffer.ColumnPhysicalType physicalType;
-    AbstractRowBuffer.ColumnLogicalType logicalType;
-    try {
-      physicalType = AbstractRowBuffer.ColumnPhysicalType.valueOf(column.getPhysicalType());
-      logicalType = AbstractRowBuffer.ColumnLogicalType.valueOf(column.getLogicalType());
-    } catch (IllegalArgumentException e) {
-      throw new SFException(
-          ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
-    }
-
-    metadata.put(Integer.toString(id), logicalType.getOrdinal() + "," + physicalType.getOrdinal());
-
-    // Parquet Type.Repetition in general supports repeated values for the same row column, like a
-    // list of values.
-    // This generator uses only either 0 or 1 value for nullable data type (OPTIONAL: 0 or none
-    // value if it is null)
-    // or exactly 1 value for non-nullable data type (REQUIRED)
-    Type.Repetition repetition =
-        column.getNullable() ? Type.Repetition.OPTIONAL : Type.Repetition.REQUIRED;
-
-    // Handle differently depends on the column logical and physical types
-    switch (logicalType) {
-      case FIXED:
-        parquetType = getFixedColumnParquetType(column, id, physicalType, repetition);
-        break;
-      case ARRAY:
-      case OBJECT:
-      case VARIANT:
-        // mark the column metadata as being an object json for the server side scanner
-        metadata.put(id + ":obj_enc", "1");
-        // parquetType is same as the next one
-      case ANY:
-      case CHAR:
-      case TEXT:
-      case BINARY:
-        parquetType =
-            Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition)
-                .as(LogicalTypeAnnotation.stringType())
-                .id(id)
-                .named(name);
-        break;
-      case TIMESTAMP_LTZ:
-      case TIMESTAMP_NTZ:
-      case TIMESTAMP_TZ:
-        parquetType =
-            getTimeColumnParquetType(
-                column.getScale(),
-                physicalType,
-                logicalType,
-                TIMESTAMP_SUPPORTED_PHYSICAL_TYPES,
-                repetition,
-                id,
-                name);
-        break;
-      case DATE:
-        parquetType =
-            Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition)
-                .as(LogicalTypeAnnotation.dateType())
-                .id(id)
-                .named(name);
-        break;
-      case TIME:
-        parquetType =
-            getTimeColumnParquetType(
-                column.getScale(),
-                physicalType,
-                logicalType,
-                TIME_SUPPORTED_PHYSICAL_TYPES,
-                repetition,
-                id,
-                name);
-        break;
-      case BOOLEAN:
-        parquetType =
-            Types.primitive(PrimitiveType.PrimitiveTypeName.BOOLEAN, repetition).id(id).named(name);
-        break;
-      case REAL:
-        parquetType =
-            Types.primitive(PrimitiveType.PrimitiveTypeName.DOUBLE, repetition).id(id).named(name);
-        break;
-      default:
-        throw new SFException(
-            ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
-    }
-    res.setParquetType(parquetType);
-    res.setMetadata(metadata);
-    return res;
-  }
-
-  /**
-   * Get the parquet type for column of Snowflake FIXED logical type.
-   *
-   * @param column column metadata
-   * @param id column id in Snowflake table schema
-   * @param physicalType Snowflake physical type of column
-   * @param repetition parquet repetition type of column
-   * @return column parquet type
-   */
-  private static Type getFixedColumnParquetType(
-      ColumnMetadata column,
-      int id,
-      AbstractRowBuffer.ColumnPhysicalType physicalType,
-      Type.Repetition repetition) {
-    String name = column.getName();
-    // the LogicalTypeAnnotation.DecimalLogicalTypeAnnotation is used by server side scanner
-    // to discover data type scale and precision
-    LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalLogicalTypeAnnotation =
-        column.getScale() != null && column.getPrecision() != null
-            ? LogicalTypeAnnotation.DecimalLogicalTypeAnnotation.decimalType(
-                column.getScale(), column.getPrecision())
-            : null;
-    Type parquetType;
-    if ((column.getScale() != null && column.getScale() != 0)
-        || physicalType == AbstractRowBuffer.ColumnPhysicalType.SB16) {
-      parquetType =
-          Types.primitive(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, repetition)
-              .length(16)
-              .as(decimalLogicalTypeAnnotation)
-              .id(id)
-              .named(name);
-    } else {
-      switch (physicalType) {
-        case SB1:
-        case SB2:
-        case SB4:
-          parquetType =
-              Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition)
-                  .as(decimalLogicalTypeAnnotation)
-                  .id(id)
-                  .length(4)
-                  .named(name);
-          break;
-        case SB8:
-          parquetType =
-              Types.primitive(PrimitiveType.PrimitiveTypeName.INT64, repetition)
-                  .as(decimalLogicalTypeAnnotation)
-                  .id(id)
-                  .length(8)
-                  .named(name);
-          break;
-        default:
-          throw new SFException(
-              ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
-      }
-    }
-    return parquetType;
-  }
-
-  /**
-   * Get the parquet type for column of a Snowflake time logical type.
-   *
-   * @param scale column scale
-   * @param physicalType Snowflake physical type of column
-   * @param logicalType Snowflake logical type of column
-   * @param supportedPhysicalTypes supported Snowflake physical types for the given column
-   * @param repetition parquet repetition type of column
-   * @param id column id in Snowflake table schema
-   * @param name column name
-   * @return column parquet type
-   */
-  private static Type getTimeColumnParquetType(
-      Integer scale,
-      AbstractRowBuffer.ColumnPhysicalType physicalType,
-      AbstractRowBuffer.ColumnLogicalType logicalType,
-      Set<AbstractRowBuffer.ColumnPhysicalType> supportedPhysicalTypes,
-      Type.Repetition repetition,
-      int id,
-      String name) {
-    if (scale == null || scale > 9 || scale < 0 || !supportedPhysicalTypes.contains(physicalType)) {
-      throw new SFException(
-          ErrorCode.UNKNOWN_DATA_TYPE,
-          "Data type: " + logicalType + ", " + physicalType + ", scale: " + scale);
-    }
-
-    PrimitiveType.PrimitiveTypeName type = getTimePrimitiveType(physicalType);
-    LogicalTypeAnnotation typeAnnotation;
-    int length;
-    switch (physicalType) {
-      case SB4:
-        typeAnnotation = LogicalTypeAnnotation.decimalType(scale, 9);
-        length = 4;
-        break;
-      case SB8:
-        typeAnnotation = LogicalTypeAnnotation.decimalType(scale, 18);
-        length = 8;
-        break;
-      case SB16:
-        typeAnnotation = LogicalTypeAnnotation.decimalType(scale, 38);
-        length = 16;
-        break;
-      default:
-        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
-    }
-    return Types.primitive(type, repetition).as(typeAnnotation).length(length).id(id).named(name);
-  }
-
-  /**
-   * Get the parquet primitive type name for column of a Snowflake time logical type.
-   *
-   * @param physicalType Snowflake physical type of column
-   * @return column parquet primitive type name
-   */
-  private static PrimitiveType.PrimitiveTypeName getTimePrimitiveType(
-      AbstractRowBuffer.ColumnPhysicalType physicalType) {
-    PrimitiveType.PrimitiveTypeName type;
-    switch (physicalType) {
-      case SB4:
-        type = PrimitiveType.PrimitiveTypeName.INT32;
-        break;
-      case SB8:
-        type = PrimitiveType.PrimitiveTypeName.INT64;
-        break;
-      case SB16:
-        type = PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY;
-        break;
-      default:
-        throw new UnsupportedOperationException("Time physical type: " + physicalType);
-    }
-    return type;
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
deleted file mode 100644
index 671af1a5b..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.math.RoundingMode;
-import java.nio.charset.StandardCharsets;
-import java.util.Optional;
-import javax.annotation.Nullable;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.parquet.schema.PrimitiveType;
-
-/** Parses a user column value into Parquet internal representation for buffering. */
-class ParquetValueParser {
-  /** Parquet internal value representation for buffering. */
-  static class ParquetBufferValue {
-    private final Object value;
-    private final float size;
-
-    ParquetBufferValue(Object value, float size) {
-      this.value = value;
-      this.size = size;
-    }
-
-    Object getValue() {
-      return value;
-    }
-
-    float getSize() {
-      return size;
-    }
-  }
-
-  /**
-   * Parses a user column value into Parquet internal representation for buffering.
-   *
-   * @param value column value provided by user in a row
-   * @param columnMetadata column metadata
-   * @param typeName Parquet primitive type name
-   * @param stats column stats to update
-   * @return parsed value and byte size of Parquet internal representation
-   */
-  static ParquetBufferValue parseColumnValueToParquet(
-      Object value,
-      ColumnMetadata columnMetadata,
-      PrimitiveType.PrimitiveTypeName typeName,
-      RowBufferStats stats) {
-    Utils.assertNotNull("Parquet column stats", stats);
-    float size = 0F;
-    if (value != null) {
-      AbstractRowBuffer.ColumnLogicalType logicalType =
-          AbstractRowBuffer.ColumnLogicalType.valueOf(columnMetadata.getLogicalType());
-      AbstractRowBuffer.ColumnPhysicalType physicalType =
-          AbstractRowBuffer.ColumnPhysicalType.valueOf(columnMetadata.getPhysicalType());
-      switch (typeName) {
-        case BOOLEAN:
-          int intValue = DataValidationUtil.validateAndParseBoolean(value);
-          value = intValue > 0;
-          stats.addIntValue(BigInteger.valueOf(intValue));
-          size = 1;
-          break;
-        case INT32:
-          int intVal =
-              getInt32Value(
-                  value,
-                  columnMetadata.getScale(),
-                  Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
-                  logicalType,
-                  physicalType);
-          value = intVal;
-          stats.addIntValue(BigInteger.valueOf(intVal));
-          size = 4;
-          break;
-        case INT64:
-          long longValue =
-              getInt64Value(
-                  value,
-                  columnMetadata.getScale(),
-                  Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
-                  logicalType,
-                  physicalType);
-          value = longValue;
-          stats.addIntValue(BigInteger.valueOf(longValue));
-          size = 8;
-          break;
-        case DOUBLE:
-          double doubleValue = DataValidationUtil.validateAndParseReal(value);
-          value = doubleValue;
-          stats.addRealValue(doubleValue);
-          size = 8;
-          break;
-        case BINARY:
-          String str = getBinaryValue(value, stats, columnMetadata);
-          value = str;
-          size = str.getBytes().length;
-          break;
-        case FIXED_LEN_BYTE_ARRAY:
-          BigInteger intRep =
-              getSb16Value(
-                  value,
-                  columnMetadata.getScale(),
-                  Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
-                  logicalType,
-                  physicalType);
-          stats.addIntValue(intRep);
-          value = getSb16Bytes(intRep);
-          size += 16;
-          break;
-        default:
-          throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
-      }
-    } else {
-      if (!columnMetadata.getNullable()) {
-        throw new SFException(
-            ErrorCode.INVALID_ROW, columnMetadata.getName(), "Passed null to non nullable field");
-      }
-      stats.incCurrentNullCount();
-    }
-    return new ParquetBufferValue(value, size);
-  }
-
-  /**
-   * Parses an int32 value based on Snowflake logical type.
-   *
-   * @param value column value provided by user in a row
-   * @param scale data type scale
-   * @param precision data type precision
-   * @param logicalType Snowflake logical type
-   * @param physicalType Snowflake physical type
-   * @return parsed int32 value
-   */
-  private static int getInt32Value(
-      Object value,
-      @Nullable Integer scale,
-      Integer precision,
-      AbstractRowBuffer.ColumnLogicalType logicalType,
-      AbstractRowBuffer.ColumnPhysicalType physicalType) {
-    int intVal;
-    switch (logicalType) {
-      case DATE:
-        intVal = DataValidationUtil.validateAndParseDate(value);
-        break;
-      case TIME:
-        Utils.assertNotNull("Unexpected null scale for TIME data type", scale);
-        intVal = DataValidationUtil.validateAndParseTime(value, scale).intValue();
-        break;
-      case FIXED:
-        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
-        bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
-        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
-        intVal = bigDecimalValue.intValue();
-        break;
-      default:
-        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
-    }
-    return intVal;
-  }
-
-  /**
-   * Parses an int64 value based on Snowflake logical type.
-   *
-   * @param value column value provided by user in a row
-   * @param scale data type scale
-   * @param precision data type precision
-   * @param logicalType Snowflake logical type
-   * @param physicalType Snowflake physical type
-   * @return parsed int64 value
-   */
-  private static long getInt64Value(
-      Object value,
-      int scale,
-      int precision,
-      AbstractRowBuffer.ColumnLogicalType logicalType,
-      AbstractRowBuffer.ColumnPhysicalType physicalType) {
-    long longValue;
-    switch (logicalType) {
-      case TIME:
-        Utils.assertNotNull("Unexpected null scale for TIME data type", scale);
-        longValue = DataValidationUtil.validateAndParseTime(value, scale).longValue();
-        break;
-      case TIMESTAMP_LTZ:
-      case TIMESTAMP_NTZ:
-        boolean ignoreTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
-        longValue =
-            DataValidationUtil.validateAndParseTimestampNtzSb16(value, scale, ignoreTimezone)
-                .getTimeInScale()
-                .longValue();
-        break;
-      case TIMESTAMP_TZ:
-        longValue =
-            DataValidationUtil.validateAndParseTimestampTz(value, scale)
-                .getSfTimestamp()
-                .orElseThrow(
-                    () ->
-                        new SFException(
-                            ErrorCode.INVALID_ROW,
-                            value,
-                            "Unable to parse timestamp for TIMESTAMP_TZ column"))
-                .toBinary(scale, true)
-                .longValue();
-        break;
-      case FIXED:
-        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
-        bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
-        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
-        longValue = bigDecimalValue.longValue();
-        break;
-      default:
-        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
-    }
-    return longValue;
-  }
-
-  /**
-   * Parses an int128 value based on Snowflake logical type.
-   *
-   * @param value column value provided by user in a row
-   * @param scale data type scale
-   * @param precision data type precision
-   * @param logicalType Snowflake logical type
-   * @param physicalType Snowflake physical type
-   * @return parsed int64 value
-   */
-  private static BigInteger getSb16Value(
-      Object value,
-      int scale,
-      int precision,
-      AbstractRowBuffer.ColumnLogicalType logicalType,
-      AbstractRowBuffer.ColumnPhysicalType physicalType) {
-    switch (logicalType) {
-      case TIMESTAMP_TZ:
-        return DataValidationUtil.validateAndParseTimestampTz(value, scale)
-            .getSfTimestamp()
-            .orElseThrow(
-                () ->
-                    new SFException(
-                        ErrorCode.INVALID_ROW,
-                        value,
-                        "Unable to parse timestamp for TIMESTAMP_TZ column"))
-            .toBinary(scale, true);
-      case TIMESTAMP_LTZ:
-      case TIMESTAMP_NTZ:
-        boolean ignoreTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
-        return DataValidationUtil.validateAndParseTimestampNtzSb16(value, scale, ignoreTimezone)
-            .getTimeInScale();
-      case FIXED:
-        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
-        // explicitly match the BigDecimal input scale with the Snowflake data type scale
-        bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
-        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
-        return bigDecimalValue.unscaledValue();
-      default:
-        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
-    }
-  }
-
-  /**
-   * Converts an int128 value to its byte array representation.
-   *
-   * @param intRep int128 value
-   * @return byte array representation
-   */
-  static byte[] getSb16Bytes(BigInteger intRep) {
-    byte[] bytes = intRep.toByteArray();
-    byte padByte = (byte) (bytes[0] < 0 ? -1 : 0);
-    byte[] bytesBE = new byte[16];
-    for (int i = 0; i < 16 - bytes.length; i++) {
-      bytesBE[i] = padByte;
-    }
-    System.arraycopy(bytes, 0, bytesBE, 16 - bytes.length, bytes.length);
-    return bytesBE;
-  }
-
-  /**
-   * Converts an object, string or binary value to its byte array representation.
-   *
-   * @param value value to parse
-   * @param stats column stats to update
-   * @param columnMetadata column metadata
-   * @return string (byte array) representation
-   */
-  private static String getBinaryValue(
-      Object value, RowBufferStats stats, ColumnMetadata columnMetadata) {
-    AbstractRowBuffer.ColumnLogicalType logicalType =
-        AbstractRowBuffer.ColumnLogicalType.valueOf(columnMetadata.getLogicalType());
-    String str;
-    if (logicalType.isObject()) {
-      switch (logicalType) {
-        case OBJECT:
-          str = DataValidationUtil.validateAndParseObject(value);
-          break;
-        case VARIANT:
-          str = DataValidationUtil.validateAndParseVariant(value);
-          break;
-        case ARRAY:
-          str = DataValidationUtil.validateAndParseArray(value);
-          break;
-        default:
-          throw new SFException(
-              ErrorCode.UNKNOWN_DATA_TYPE, logicalType, columnMetadata.getPhysicalType());
-      }
-    } else if (logicalType == AbstractRowBuffer.ColumnLogicalType.BINARY) {
-      String maxLengthString = columnMetadata.getLength().toString();
-      byte[] bytes =
-          DataValidationUtil.validateAndParseBinary(
-              value, Optional.of(maxLengthString).map(Integer::parseInt));
-      str = new String(bytes, StandardCharsets.UTF_8);
-      stats.addStrValue(str);
-    } else {
-      String maxLengthString = columnMetadata.getLength().toString();
-      str =
-          DataValidationUtil.validateAndParseString(
-              value, Optional.of(maxLengthString).map(Integer::parseInt));
-      stats.addStrValue(str);
-    }
-    return str;
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index 44b90987b..739ae5499 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -25,8 +25,7 @@
  * Register one or more blobs to the targeted Snowflake table, it will be done using the dedicated
  * thread in order to maintain ordering per channel
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
 class RegisterService<T> {
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
index 2aa9d9861..a229f4db3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
@@ -13,8 +13,7 @@
  * Interface for the buffer in the Streaming Ingest channel that holds the un-flushed rows, these
  * rows will be converted to the underlying format implementation for faster processing
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
 interface RowBuffer<T> {
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 7016b23f5..d2d64db29 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -30,8 +30,7 @@
 /**
  * The first version of implementation for SnowflakeStreamingIngestChannel
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
 class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIngestChannel {
 
@@ -149,20 +148,9 @@ private RowBuffer<T> createRowBuffer(Constants.BdecVersion bdecVersion) {
     // TODO: The circular dependency SnowflakeStreamingIngestChannelInternal <-> RowBuffer
     // (SNOW-657667)
     // can be probably reconsidered
-    switch (bdecVersion) {
-      case ONE:
-      case TWO:
-        //noinspection unchecked
-        return (RowBuffer<T>)
-            new ArrowRowBuffer((SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>) this);
-      case THREE:
-        //noinspection unchecked
-        return (RowBuffer<T>)
-            new ParquetRowBuffer((SnowflakeStreamingIngestChannelInternal<ParquetChunkData>) this);
-      default:
-        throw new SFException(
-            ErrorCode.INTERNAL_ERROR, "Unsupported BDEC format version: " + bdecVersion);
-    }
+    //noinspection unchecked
+    return (RowBuffer<T>)
+        new ArrowRowBuffer((SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>) this);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 8be3c4ed3..6a3e99382 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -82,8 +82,7 @@
  * <li>the channel cache, which contains all the channels that belong to this account
  * <li>the flush service, which schedules and coordinates the flush to Snowflake tables
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
 public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStreamingIngestClient {
 
@@ -554,7 +553,7 @@ List<BlobMetadata> getRetryBlobs(
                   .collect(Collectors.toList());
           if (!relevantChunks.isEmpty()) {
             retryBlobs.add(
-                BlobMetadata.createBlobMetadata(
+                new BlobMetadata(
                     blobMetadata.getPath(),
                     blobMetadata.getMD5(),
                     blobMetadata.getVersion(),
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index 68602e0a9..fed4022e1 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -83,14 +83,7 @@ public enum BdecVersion {
     ONE(1),
 
     /** Uses Arrow to generate BDEC chunks with {@link ArrowBatchWriteMode#FILE}. */
-    TWO(2),
-
-    /**
-     * Uses Parquet to generate BDEC chunks with {@link
-     * net.snowflake.ingest.streaming.internal.ParquetRowBuffer} (page-level compression). This
-     * version is experimental and WIP at the moment.
-     */
-    THREE(3);
+    TWO(2);
 
     private final byte version;
 
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 9f586a04c..5e5e594cd 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -14,6 +14,7 @@
 import static net.snowflake.ingest.utils.Constants.USER;
 import static net.snowflake.ingest.utils.Constants.WAREHOUSE;
 import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION;
+import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -34,7 +35,6 @@
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
-import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.Utils;
 import org.apache.commons.codec.binary.Base64;
 import org.junit.Assert;
@@ -180,7 +180,7 @@ public static KeyPair getKeyPair() throws Exception {
     return keyPair;
   }
 
-  public static Properties getProperties(Constants.BdecVersion bdecVersion) throws Exception {
+  public static Properties getProperties() throws Exception {
     if (profile == null) {
       init();
     }
@@ -195,10 +195,16 @@ public static Properties getProperties(Constants.BdecVersion bdecVersion) throws
     props.put(PRIVATE_KEY, privateKeyPem);
     props.put(ROLE, role);
     props.put(ACCOUNT_URL, getAccountURL());
-    props.put(BLOB_FORMAT_VERSION, bdecVersion.toByte());
+    props.put(BLOB_FORMAT_VERSION, getBlobFormatVersion());
     return props;
   }
 
+  private static byte getBlobFormatVersion() {
+    return profile.has(BLOB_FORMAT_VERSION)
+        ? (byte) profile.get(BLOB_FORMAT_VERSION).asInt()
+        : BLOB_FORMAT_VERSION_DEFAULT.toByte();
+  }
+
   /**
    * Create snowflake jdbc connection
    *
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
index 5785bd221..c07966c68 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
@@ -57,13 +57,15 @@ public void testFieldNumberAfterFlush() {
   @Test
   public void buildFieldFixedSB1() {
     // FIXED, SB1
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB1")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB1");
+    testCol.setNullable(true);
+    testCol.setLogicalType("FIXED");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -78,13 +80,15 @@ public void buildFieldFixedSB1() {
 
   @Test
   public void buildFieldFixedSB2() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB2")
-            .nullable(false)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB2");
+    testCol.setNullable(false);
+    testCol.setLogicalType("FIXED");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -99,13 +103,15 @@ public void buildFieldFixedSB2() {
 
   @Test
   public void buildFieldFixedSB4() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB4")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB4");
+    testCol.setNullable(true);
+    testCol.setLogicalType("FIXED");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -119,13 +125,15 @@ public void buildFieldFixedSB4() {
 
   @Test
   public void buildFieldFixedSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB8");
+    testCol.setNullable(true);
+    testCol.setLogicalType("FIXED");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -139,13 +147,15 @@ public void buildFieldFixedSB8() {
 
   @Test
   public void buildFieldFixedSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB16");
+    testCol.setNullable(true);
+    testCol.setLogicalType("FIXED");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     ArrowType expectedType =
@@ -162,13 +172,15 @@ public void buildFieldFixedSB16() {
 
   @Test
   public void buildFieldLobVariant() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("VARIANT")
-            .physicalType("LOB")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("LOB");
+    testCol.setNullable(true);
+    testCol.setLogicalType("VARIANT");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -182,13 +194,15 @@ public void buildFieldLobVariant() {
 
   @Test
   public void buildFieldTimestampNtzSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_NTZ")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB8");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIMESTAMP_NTZ");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -202,13 +216,15 @@ public void buildFieldTimestampNtzSB8() {
 
   @Test
   public void buildFieldTimestampNtzSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_NTZ")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB16");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIMESTAMP_NTZ");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -226,13 +242,15 @@ public void buildFieldTimestampNtzSB16() {
 
   @Test
   public void buildFieldTimestampTzSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_TZ")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB8");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIMESTAMP_TZ");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -250,13 +268,15 @@ public void buildFieldTimestampTzSB8() {
 
   @Test
   public void buildFieldTimestampTzSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_TZ")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB16");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIMESTAMP_TZ");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -275,14 +295,16 @@ public void buildFieldTimestampTzSB16() {
   }
 
   @Test
-  public void buildFieldTimestampDate() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("DATE")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
+  public void buildFieldDate() {
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB8");
+    testCol.setNullable(true);
+    testCol.setLogicalType("DATE");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -296,13 +318,15 @@ public void buildFieldTimestampDate() {
 
   @Test
   public void buildFieldTimeSB4() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIME")
-            .physicalType("SB4")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB4");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIME");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -316,13 +340,15 @@ public void buildFieldTimeSB4() {
 
   @Test
   public void buildFieldTimeSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIME")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB8");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIME");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -336,13 +362,15 @@ public void buildFieldTimeSB8() {
 
   @Test
   public void buildFieldBoolean() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("BOOLEAN")
-            .physicalType("BINARY")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("BINARY");
+    testCol.setNullable(true);
+    testCol.setLogicalType("BOOLEAN");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -356,13 +384,15 @@ public void buildFieldBoolean() {
 
   @Test
   public void buildFieldRealSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("REAL")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB16");
+    testCol.setNullable(true);
+    testCol.setLogicalType("REAL");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java b/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
deleted file mode 100644
index 51d4226bb..000000000
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-/** Helper class to build ColumnMetadata */
-public class ColumnMetadataBuilder {
-  private String name;
-  private String type;
-  private String logicalType;
-  private String physicalType;
-  private Integer precision;
-  private Integer scale;
-  private Integer byteLength;
-  private Integer length;
-  private boolean nullable;
-  private String collation;
-
-  /**
-   * Returns a new ColumnMetadata builder
-   *
-   * @return builder
-   */
-  public static ColumnMetadataBuilder newBuilder() {
-    ColumnMetadataBuilder mb = new ColumnMetadataBuilder();
-    mb.name = "testCol";
-    mb.byteLength = 14;
-    mb.length = 11;
-    mb.scale = 0;
-    mb.precision = 4;
-    return mb;
-  }
-
-  /**
-   * Set name
-   *
-   * @param name column name
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder name(String name) {
-    this.name = name;
-    return this;
-  }
-
-  /**
-   * Set type
-   *
-   * @param type type (as defined in ColumnMetadata)
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder type(String type) {
-    this.type = type;
-    return this;
-  }
-
-  /**
-   * Set column logical type
-   *
-   * @param logicalType logical type
-   * @return columnMetadata object
-   */
-  public ColumnMetadataBuilder logicalType(String logicalType) {
-    this.logicalType = logicalType;
-    return this;
-  }
-
-  /**
-   * Set column physical type
-   *
-   * @param physicalType physical type
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder physicalType(String physicalType) {
-    this.physicalType = physicalType;
-    return this;
-  }
-
-  /**
-   * Set column precision
-   *
-   * @param precision precision
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder precision(int precision) {
-    this.precision = precision;
-    return this;
-  }
-
-  /**
-   * Set column scale
-   *
-   * @param scale scale
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder scale(int scale) {
-    this.scale = scale;
-    return this;
-  }
-
-  /**
-   * Set column length in bytes
-   *
-   * @param byteLength length in bytes
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder byteLength(int byteLength) {
-    this.byteLength = byteLength;
-    return this;
-  }
-
-  /**
-   * Set column length (in chars)
-   *
-   * @param length length
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder length(int length) {
-    this.length = length;
-    return this;
-  }
-
-  /**
-   * Set column nullability
-   *
-   * @param nullable nullable
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder nullable(boolean nullable) {
-    this.nullable = nullable;
-    return this;
-  }
-
-  /**
-   * Set column collation
-   *
-   * @param collation collation
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder collation(String collation) {
-    this.collation = collation;
-    return this;
-  }
-
-  /**
-   * Build a columnMetadata object from the set values
-   *
-   * @return columnMetadata object
-   */
-  public ColumnMetadata build() {
-    ColumnMetadata colMetadata = new ColumnMetadata();
-    colMetadata.setName(name);
-    colMetadata.setType(type);
-    colMetadata.setPhysicalType(physicalType);
-    colMetadata.setNullable(nullable);
-    colMetadata.setLogicalType(logicalType);
-    colMetadata.setByteLength(byteLength);
-    colMetadata.setLength(length);
-    colMetadata.setScale(scale);
-    colMetadata.setPrecision(precision);
-    colMetadata.setCollation(collation);
-    return colMetadata;
-  }
-}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 540cd5ea8..80103e2df 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -55,8 +55,7 @@
 public class FlushServiceTest {
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> testContextFactory() {
-    return Arrays.asList(
-        new Object[][] {{ArrowTestContext.createFactory()}, {ParquetTestContext.createFactory()}});
+    return Arrays.asList(new Object[][] {{ArrowTestContext.createFactory()}});
   }
 
   public FlushServiceTest(TestContextFactory<?> testContextFactory) {
@@ -183,6 +182,11 @@ ChannelBuilder setEncryptionKeyId(Long encryptionKeyId) {
         return this;
       }
 
+      ChannelBuilder setOnErrorOption(OpenChannelRequest.OnErrorOption onErrorOption) {
+        this.onErrorOption = onErrorOption;
+        return this;
+      }
+
       SnowflakeStreamingIngestChannelInternal<T> buildAndAdd() {
         SnowflakeStreamingIngestChannelInternal<T> channel =
             createChannel(
@@ -281,48 +285,6 @@ TestContext<VectorSchemaRoot> create() {
     }
   }
 
-  private static class ParquetTestContext extends TestContext<List<List<Object>>> {
-
-    SnowflakeStreamingIngestChannelInternal<List<List<Object>>> createChannel(
-        String name,
-        String dbName,
-        String schemaName,
-        String tableName,
-        String offsetToken,
-        Long channelSequencer,
-        Long rowSequencer,
-        String encryptionKey,
-        Long encryptionKeyId,
-        OpenChannelRequest.OnErrorOption onErrorOption) {
-      return new SnowflakeStreamingIngestChannelInternal<>(
-          name,
-          dbName,
-          schemaName,
-          tableName,
-          offsetToken,
-          channelSequencer,
-          rowSequencer,
-          client,
-          encryptionKey,
-          encryptionKeyId,
-          onErrorOption,
-          Constants.BdecVersion.THREE,
-          null);
-    }
-
-    @Override
-    public void close() {}
-
-    static TestContextFactory<List<List<Object>>> createFactory() {
-      return new TestContextFactory<List<List<Object>>>("Parquet") {
-        @Override
-        TestContext<List<List<Object>>> create() {
-          return new ParquetTestContext();
-        }
-      };
-    }
-  }
-
   TestContextFactory<?> testContextFactory;
   TestContext<?> testContext;
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
deleted file mode 100644
index a477ac977..000000000
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
+++ /dev/null
@@ -1,480 +0,0 @@
-package net.snowflake.ingest.streaming.internal;
-
-import java.util.Map;
-import org.apache.parquet.schema.LogicalTypeAnnotation;
-import org.apache.parquet.schema.PrimitiveType;
-import org.apache.parquet.schema.Type;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class ParquetTypeGeneratorTest {
-
-  @Test
-  public void buildFieldFixedSB1() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB1")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(4)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
-        .expectedLogicalTypeAnnotation(
-            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB1.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldFixedSB2() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB2")
-            .nullable(false)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(4)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
-        .expectedLogicalTypeAnnotation(
-            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
-        .expectedRepetition(Type.Repetition.REQUIRED)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB2.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldFixedSB4() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB4")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(4)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
-        .expectedLogicalTypeAnnotation(
-            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB4.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldFixedSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(8)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
-        .expectedLogicalTypeAnnotation(
-            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldFixedSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(16)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
-        .expectedLogicalTypeAnnotation(
-            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldLobVariant() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("VARIANT")
-            .physicalType("LOB")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(0)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.BINARY)
-        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.stringType())
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.VARIANT.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.LOB.getOrdinal())
-        .assertMatches();
-
-    Assert.assertEquals("1", typeInfo.getMetadata().get(0 + ":obj_enc"));
-  }
-
-  @Test
-  public void buildFieldTimestampNtzSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_NTZ")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(8)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
-        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldTimestampNtzSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_NTZ")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(16)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
-        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 38))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldTimestampTzSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_TZ")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(8)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
-        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_TZ.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldTimestampTzSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_TZ")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(16)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
-        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 38))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_TZ.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldDate() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("DATE")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(0)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
-        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.dateType())
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.DATE.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldTimeSB4() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIME")
-            .physicalType("SB4")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(4)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
-        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 9))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.TIME.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB4.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldTimeSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIME")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(8)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
-        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.TIME.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldBoolean() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("BOOLEAN")
-            .physicalType("BINARY")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(0)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.BOOLEAN)
-        .expectedLogicalTypeAnnotation(null)
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.BOOLEAN.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.BINARY.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldRealSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("REAL")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(0)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.DOUBLE)
-        .expectedLogicalTypeAnnotation(null)
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.REAL.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
-        .assertMatches();
-  }
-
-  /** Builder that helps to assert parquet type info */
-  private static class ParquetTypeInfoAssertionBuilder {
-    private String fieldName;
-    private int fieldId;
-    private PrimitiveType.PrimitiveTypeName primitiveTypeName;
-    private LogicalTypeAnnotation logicalTypeAnnotation;
-    private Type.Repetition repetition;
-    private int typeLength;
-    private String colMetadata;
-    private ParquetTypeGenerator.ParquetTypeInfo typeInfo;
-
-    static ParquetTypeInfoAssertionBuilder newBuilder() {
-      ParquetTypeInfoAssertionBuilder builder = new ParquetTypeInfoAssertionBuilder();
-      return builder;
-    }
-
-    ParquetTypeInfoAssertionBuilder typeInfo(ParquetTypeGenerator.ParquetTypeInfo typeInfo) {
-      this.typeInfo = typeInfo;
-      return this;
-    }
-
-    ParquetTypeInfoAssertionBuilder expectedFieldName(String fieldName) {
-      this.fieldName = fieldName;
-      return this;
-    }
-
-    ParquetTypeInfoAssertionBuilder expectedFieldId(int fieldId) {
-      this.fieldId = fieldId;
-      return this;
-    }
-
-    ParquetTypeInfoAssertionBuilder expectedPrimitiveTypeName(
-        PrimitiveType.PrimitiveTypeName primitiveTypeName) {
-      this.primitiveTypeName = primitiveTypeName;
-      return this;
-    }
-
-    ParquetTypeInfoAssertionBuilder expectedLogicalTypeAnnotation(
-        LogicalTypeAnnotation logicalTypeAnnotation) {
-      this.logicalTypeAnnotation = logicalTypeAnnotation;
-      return this;
-    }
-
-    ParquetTypeInfoAssertionBuilder expectedRepetition(Type.Repetition repetition) {
-      this.repetition = repetition;
-      return this;
-    }
-
-    ParquetTypeInfoAssertionBuilder expectedTypeLength(int typeLength) {
-      this.typeLength = typeLength;
-      return this;
-    }
-
-    ParquetTypeInfoAssertionBuilder expectedColMetaData(String colMetaData) {
-      this.colMetadata = colMetaData;
-      return this;
-    }
-
-    void assertMatches() {
-      Type type = typeInfo.getParquetType();
-      Map<String, String> metadata = typeInfo.getMetadata();
-      Assert.assertEquals(fieldName, type.getName());
-      Assert.assertEquals(typeLength, type.asPrimitiveType().getTypeLength());
-      Assert.assertEquals(fieldId, type.asPrimitiveType().getId().intValue());
-
-      Assert.assertEquals(primitiveTypeName, type.asPrimitiveType().getPrimitiveTypeName());
-      Assert.assertEquals(logicalTypeAnnotation, type.getLogicalTypeAnnotation());
-      Assert.assertEquals(repetition, type.getRepetition());
-      Assert.assertEquals(colMetadata, metadata.get(type.getId().toString()));
-    }
-  }
-}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
deleted file mode 100644
index 3dc313dc6..000000000
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
+++ /dev/null
@@ -1,570 +0,0 @@
-package net.snowflake.ingest.streaming.internal;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.HashMap;
-import java.util.Map;
-import net.snowflake.ingest.utils.SFException;
-import org.apache.parquet.schema.PrimitiveType;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class ParquetValueParserTest {
-
-  @Test
-  public void parseValueFixedSB1ToInt32() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB1")
-            .scale(0)
-            .precision(2)
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            12, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Integer.class)
-        .expectedParsedValue(12)
-        .expectedSize(4.0f)
-        .expectedMinMax(BigInteger.valueOf(12))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueFixedSB2ToInt32() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB2")
-            .scale(0)
-            .precision(4)
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            1234, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Integer.class)
-        .expectedParsedValue(1234)
-        .expectedSize(4.0f)
-        .expectedMinMax(BigInteger.valueOf(1234))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueFixedSB4ToInt32() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB4")
-            .scale(0)
-            .precision(9)
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            123456789, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Integer.class)
-        .expectedParsedValue(123456789)
-        .expectedSize(4.0f)
-        .expectedMinMax(BigInteger.valueOf(123456789))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueFixedSB8ToInt64() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB8")
-            .scale(0)
-            .precision(18)
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            123456789987654321L, testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Long.class)
-        .expectedParsedValue(123456789987654321L)
-        .expectedSize(8.0f)
-        .expectedMinMax(BigInteger.valueOf(123456789987654321L))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueFixedSB16ToByteArray() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB16")
-            .scale(0)
-            .precision(38)
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            new BigDecimal("91234567899876543219876543211234567891"),
-            testCol,
-            PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
-            rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(byte[].class)
-        .expectedParsedValue(
-            ParquetValueParser.getSb16Bytes(
-                new BigInteger("91234567899876543219876543211234567891")))
-        .expectedSize(16.0f)
-        .expectedMinMax(new BigInteger("91234567899876543219876543211234567891"))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueFixedDecimalToInt32() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB8")
-            .scale(5)
-            .precision(10)
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            new BigDecimal("12345.54321"),
-            testCol,
-            PrimitiveType.PrimitiveTypeName.DOUBLE,
-            rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Double.class)
-        .expectedParsedValue(Double.valueOf("12345.54321"))
-        .expectedSize(8.0f)
-        .expectedMinMax(Double.valueOf("12345.54321"))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueDouble() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("REAL")
-            .physicalType("DOUBLE")
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            12345.54321d, testCol, PrimitiveType.PrimitiveTypeName.DOUBLE, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Double.class)
-        .expectedParsedValue(Double.valueOf(12345.54321))
-        .expectedSize(8.0f)
-        .expectedMinMax(Double.valueOf(12345.54321))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueBoolean() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("BOOLEAN")
-            .physicalType("SB1")
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            true, testCol, PrimitiveType.PrimitiveTypeName.BOOLEAN, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Boolean.class)
-        .expectedParsedValue(true)
-        .expectedSize(1.0f)
-        .expectedMinMax(BigInteger.valueOf(1))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueBinary() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("BINARY")
-            .physicalType("LOB")
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            "Length7".getBytes(), testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(String.class)
-        .expectedParsedValue("Length7")
-        .expectedSize(7.0f)
-        .expectedMinMax("Length7")
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueVariantToBinary() {
-    testJsonWithLogicalType("VARIANT");
-  }
-
-  @Test
-  public void parseValueObjectToBinary() {
-    testJsonWithLogicalType("OBJECT");
-  }
-
-  private void testJsonWithLogicalType(String logicalType) {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType(logicalType)
-            .physicalType("BINARY")
-            .nullable(true)
-            .build();
-
-    String var =
-        "{\"key1\":-879869596,\"key2\":\"value2\",\"key3\":null,"
-            + "\"key4\":{\"key41\":0.032437,\"key42\":\"value42\",\"key43\":null}}";
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            var, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(String.class)
-        .expectedParsedValue(var)
-        .expectedSize(var.getBytes().length)
-        .expectedMinMax(null)
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueArrayToBinary() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("ARRAY")
-            .physicalType("BINARY")
-            .nullable(true)
-            .build();
-
-    Map<String, String> input = new HashMap<>();
-    input.put("a", "1");
-    input.put("b", "2");
-    input.put("c", "3");
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            input, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
-
-    String resultArray = "[{\"a\":\"1\",\"b\":\"2\",\"c\":\"3\"}]";
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(String.class)
-        .expectedParsedValue(resultArray)
-        .expectedSize(resultArray.length())
-        .expectedMinMax(null)
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueTimestampNtzSB4Error() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_NTZ")
-            .physicalType("SB4")
-            .scale(0) // seconds
-            .precision(9)
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    SFException exception =
-        Assert.assertThrows(
-            SFException.class,
-            () ->
-                ParquetValueParser.parseColumnValueToParquet(
-                    "2013-04-28 20:57:00",
-                    testCol,
-                    PrimitiveType.PrimitiveTypeName.INT32,
-                    rowBufferStats));
-    Assert.assertEquals(
-        "Unknown data type for logical: TIMESTAMP_NTZ, physical: SB4.", exception.getMessage());
-  }
-
-  @Test
-  public void parseValueTimestampNtzSB8ToINT64() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_NTZ")
-            .physicalType("SB8")
-            .scale(3) // millis
-            .precision(18)
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            "2013-04-28 20:57:01.000",
-            testCol,
-            PrimitiveType.PrimitiveTypeName.INT64,
-            rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Long.class)
-        .expectedParsedValue(1367182621000L)
-        .expectedSize(8.0f)
-        .expectedMinMax(BigInteger.valueOf(1367182621000L))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueTimestampNtzSB16ToByteArray() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_NTZ")
-            .physicalType("SB16")
-            .nullable(true)
-            .scale(9) // nanos
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            "2022-09-18 22:05:07.123456789",
-            testCol,
-            PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
-            rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(byte[].class)
-        .expectedParsedValue(
-            ParquetValueParser.getSb16Bytes(BigInteger.valueOf(1663538707123456789L)))
-        .expectedSize(16.0f)
-        .expectedMinMax(BigInteger.valueOf(1663538707123456789L))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueDateToInt32() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("DATE")
-            .physicalType("SB4")
-            .scale(0) // seconds
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            "2021-01-01", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Integer.class)
-        .expectedParsedValue(Integer.valueOf(18628))
-        .expectedSize(4.0f)
-        .expectedMinMax(BigInteger.valueOf(18628))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueTimeSB4ToInt32() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIME")
-            .physicalType("SB4")
-            .scale(0) // seconds
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            "01:00:00", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Integer.class)
-        .expectedParsedValue(3600)
-        .expectedSize(4.0f)
-        .expectedMinMax(BigInteger.valueOf(3600))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueTimeSB8ToInt64() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIME")
-            .physicalType("SB8")
-            .scale(3) // milliseconds
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            "01:00:00.123", testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Long.class)
-        .expectedParsedValue(3600123L)
-        .expectedSize(8.0f)
-        .expectedMinMax(BigInteger.valueOf(3600123))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueTimeSB16Error() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIME")
-            .physicalType("SB16")
-            .scale(9) // nanos
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    SFException exception =
-        Assert.assertThrows(
-            SFException.class,
-            () ->
-                ParquetValueParser.parseColumnValueToParquet(
-                    "11:00:00.12345678",
-                    testCol,
-                    PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
-                    rowBufferStats));
-    Assert.assertEquals(
-        "Unknown data type for logical: TIME, physical: SB16.", exception.getMessage());
-  }
-
-  /** Builder that helps to assert parsing of values to parquet types */
-  private static class ParquetValueParserAssertionBuilder {
-    private ParquetValueParser.ParquetBufferValue parquetBufferValue;
-    private RowBufferStats rowBufferStats;
-    private Class valueClass;
-    private Object value;
-    private float size;
-    private Object minMaxStat;
-
-    static ParquetValueParserAssertionBuilder newBuilder() {
-      ParquetValueParserAssertionBuilder builder = new ParquetValueParserAssertionBuilder();
-      return builder;
-    }
-
-    ParquetValueParserAssertionBuilder parquetBufferValue(
-        ParquetValueParser.ParquetBufferValue parquetBufferValue) {
-      this.parquetBufferValue = parquetBufferValue;
-      return this;
-    }
-
-    ParquetValueParserAssertionBuilder rowBufferStats(RowBufferStats rowBufferStats) {
-      this.rowBufferStats = rowBufferStats;
-      return this;
-    }
-
-    ParquetValueParserAssertionBuilder expectedValueClass(Class valueClass) {
-      this.valueClass = valueClass;
-      return this;
-    }
-
-    ParquetValueParserAssertionBuilder expectedParsedValue(Object value) {
-      this.value = value;
-      return this;
-    }
-
-    ParquetValueParserAssertionBuilder expectedSize(float size) {
-      this.size = size;
-      return this;
-    }
-
-    public ParquetValueParserAssertionBuilder expectedMinMax(Object minMaxStat) {
-      this.minMaxStat = minMaxStat;
-      return this;
-    }
-
-    void assertMatches() {
-      Assert.assertEquals(valueClass, parquetBufferValue.getValue().getClass());
-      System.out.println("parquetBufferValue = " + parquetBufferValue.getValue().toString());
-      if (valueClass.equals(byte[].class)) {
-        Assert.assertArrayEquals((byte[]) value, (byte[]) parquetBufferValue.getValue());
-      } else {
-        Assert.assertEquals(value, parquetBufferValue.getValue());
-      }
-      Assert.assertEquals(size, parquetBufferValue.getSize(), 0);
-      if (minMaxStat instanceof BigInteger) {
-        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinIntValue());
-        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxIntValue());
-        return;
-      } else if (minMaxStat instanceof String || valueClass.equals(String.class)) {
-        // String can have null min/max stats for variant data types
-        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinColStrValue());
-        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxColStrValue());
-        return;
-      } else if (minMaxStat instanceof Double || minMaxStat instanceof BigDecimal) {
-        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinRealValue());
-        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxRealValue());
-        return;
-      }
-      throw new IllegalArgumentException("Unknown data type for min stat");
-    }
-  }
-}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 8d28c0cb9..7bd986e8a 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -26,11 +26,7 @@
 public class RowBufferTest {
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> bdecVersion() {
-    return Arrays.asList(
-        new Object[][] {
-          {"Arrow", Constants.BdecVersion.ONE},
-          {"Parquet", Constants.BdecVersion.THREE}
-        });
+    return Arrays.asList(new Object[][] {{"Arrow", Constants.BdecVersion.ONE}});
   }
 
   private final Constants.BdecVersion bdecVersion;
@@ -1033,7 +1029,6 @@ private void testE2EBinaryHelper(OpenChannelRequest.OnErrorOption onErrorOption)
     colBinary.setPhysicalType("LOB");
     colBinary.setNullable(true);
     colBinary.setLogicalType("BINARY");
-    colBinary.setLength(32);
     colBinary.setScale(0);
 
     innerBuffer.setupSchema(Collections.singletonList(colBinary));
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index c5189bcde..175db5fb8 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -9,15 +9,14 @@
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Random;
 import java.util.TimeZone;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
@@ -32,7 +31,6 @@
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
-import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.ParameterProvider;
 import net.snowflake.ingest.utils.SFException;
@@ -40,23 +38,9 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 
 /** Example streaming ingest sdk integration test */
-@RunWith(Parameterized.class)
 public class StreamingIngestIT {
-  @Parameterized.Parameters(name = "{0}")
-  public static Collection<Object[]> bdecVersion() {
-    return Arrays.asList(
-        new Object[][] {
-          {"Arrow", Constants.BdecVersion.ONE},
-          // TODO: uncomment once SNOW-659721 is deployed and we set the parameter
-          // DISABLE_PARQUET_CACHE to true for the test account
-          // {"Parquet", Constants.BdecVersion.THREE}
-        });
-  }
-
   private static final String TEST_TABLE = "STREAMING_INGEST_TEST_TABLE";
   private static final String TEST_DB_PREFIX = "STREAMING_INGEST_TEST_DB";
   private static final String TEST_SCHEMA = "STREAMING_INGEST_TEST_SCHEMA";
@@ -71,23 +55,11 @@ public static Collection<Object[]> bdecVersion() {
   private Connection jdbcConnection;
   private String testDb;
 
-  private final Constants.BdecVersion bdecVersion;
-
-  public StreamingIngestIT(
-      @SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
-    this.bdecVersion = bdecVersion;
-  }
-
   @Before
   public void beforeAll() throws Exception {
     testDb = TEST_DB_PREFIX + "_" + UUID.randomUUID().toString().substring(0, 4);
     // Create a streaming ingest client
     jdbcConnection = TestUtils.getConnection(true);
-    if (bdecVersion == Constants.BdecVersion.THREE) {
-      // TODO: encryption and interleaved mode are not yet supported by server side's Parquet
-      // scanner if local file cache is enabled (SNOW-656500)
-      jdbcConnection.createStatement().execute("alter session set disable_parquet_cache=true;");
-    }
     jdbcConnection
         .createStatement()
         .execute(String.format("create or replace database %s;", testDb));
@@ -106,7 +78,7 @@ public void beforeAll() throws Exception {
         .createStatement()
         .execute(String.format("use warehouse %s", TestUtils.getWarehouse()));
 
-    prop = TestUtils.getProperties(bdecVersion);
+    prop = TestUtils.getProperties();
     if (prop.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       prop.setProperty(ROLE, "ACCOUNTADMIN");
     }
@@ -1070,7 +1042,7 @@ private void verifyTableRowCount(int rowNumber, String tableName) {
   }
 
   @Test
-  public void testDataTypes() throws SQLException, ParseException {
+  public void testDataTypes() {
     String tableName = "t_data_types";
     try {
       jdbcConnection
@@ -1108,195 +1080,94 @@ public void testDataTypes() throws SQLException, ParseException {
 
     SnowflakeStreamingIngestChannel channel = openChannel(tableName, "CHANNEL");
 
-    Map<String, Object> negRow = getNegRow();
-    verifyInsertValidationResponse(channel.insertRow(negRow, Integer.toString(0)));
-
-    Map<String, Object> nullRow = getNullRow();
-    verifyInsertValidationResponse(channel.insertRow(nullRow, Integer.toString(1)));
-
-    Map<String, Object> posRow = getPosRow();
-    verifyInsertValidationResponse(channel.insertRow(posRow, Integer.toString(2)));
-
-    waitChannelFlushed(channel, 3);
-    verifyTableRowCount(3, tableName);
-
-    ResultSet result =
-        jdbcConnection
-            .createStatement()
-            .executeQuery(
-                String.format(
-                    "select * from %s.%s.%s order by num", testDb, TEST_SCHEMA, tableName));
-
-    result.next(); // negRow
-    assertNegRow(negRow, result);
-
-    result.next(); // nullRow
-    assertNullRow(nullRow, result);
-
-    result.next(); // posRow
-    assertPosRow(posRow, result);
-  }
-
-  private Map<String, Object> getNegRow() {
-    Map<String, Object> negRow = new HashMap<>();
-    negRow.put("num", -123);
-    negRow.put("num_10_5", new BigDecimal("-12345.54321"));
-    negRow.put("num_38_0", new BigDecimal("-91234567899876543219876543211234567891"));
-    negRow.put("num_2_0", -12);
-    negRow.put("num_4_0", -1234);
-    negRow.put("num_9_0", -123456789);
-    negRow.put("num_18_0", -123456789987654321L);
-    negRow.put("num_float", -12345.54321f);
-    negRow.put("str_varchar", "zxccvbnm,.");
-    negRow.put("bin", new byte[] {1, 2, 3});
-    negRow.put("bl", true);
-    negRow.put("var", "{}");
-    negRow.put("obj", "{}");
-    negRow.put("arr", Arrays.asList());
-    negRow.put("epochdays", "0");
-    negRow.put("epochsec", "0");
-    negRow.put("epochnano", "0");
-    negRow.put("timesec", "0");
-    negRow.put("timenano", "0");
-
-    return negRow;
+    Random r = new Random();
+    for (int val = 0; val < 10000; val++) {
+      verifyInsertValidationResponse(channel.insertRow(getRandomRow(r), Integer.toString(val)));
+    }
+    waitChannelFlushed(channel, 10000);
   }
 
-  private Map<String, Object> getNullRow() {
-    Map<String, Object> nullRow = new HashMap<>();
-    nullRow.put("num", 0);
-    nullRow.put("num_10_5", new BigDecimal("0.00000"));
-    nullRow.put("num_38_0", new BigDecimal("0"));
-    nullRow.put("num_2_0", 0);
-    nullRow.put("num_4_0", 0);
-    nullRow.put("num_9_0", 0);
-    nullRow.put("num_18_0", 0L);
-    nullRow.put("num_float", 0.0f);
-    nullRow.put("str_varchar", null);
-    nullRow.put("bin", null);
-    nullRow.put("bl", false);
-    nullRow.put("var", null);
-    nullRow.put("obj", null);
-    nullRow.put("arr", null);
-    nullRow.put("epochdays", null);
-    nullRow.put("epochsec", null);
-    nullRow.put("epochnano", null);
-    nullRow.put("timesec", null);
-    nullRow.put("timenano", null);
-
-    return nullRow;
+  private static Map<String, Object> getRandomRow(Random r) {
+    Map<String, Object> row = new HashMap<>();
+    row.put("num", r.nextInt());
+    row.put("num_10_5", nextFloat(r));
+    row.put(
+        "num_38_0",
+        new BigDecimal("" + nextLongOfPrecision(r, 18) + Math.abs(nextLongOfPrecision(r, 18))));
+    row.put("num_2_0", r.nextInt(100));
+    row.put("num_4_0", r.nextInt(10000));
+    row.put("num_9_0", r.nextInt(1000000000));
+    row.put("num_18_0", nextLongOfPrecision(r, 18));
+    row.put("num_float", nextFloat(r));
+    row.put("str_varchar", nextString(r));
+    row.put("bin", nextBytes(r));
+    row.put("bl", r.nextBoolean());
+    row.put("var", nextJson(r));
+    row.put("obj", nextJson(r));
+    row.put("arr", Arrays.asList(r.nextInt(100), r.nextInt(100), r.nextInt(100)));
+    row.put(
+        "epochdays",
+        String.valueOf(Math.abs(r.nextInt()) % (18963 * 24 * 60 * 60))); // DATE, 02.12.2021
+    row.put(
+        "timesec",
+        String.valueOf(
+            (r.nextInt(11) * 60 * 60 + r.nextInt(59) * 60 + r.nextInt(59)) * 10000
+                + r.nextInt(9999))); // TIME(4), 05:12:43.4536
+    row.put(
+        "timenano",
+        String.valueOf(
+            (14 * 60 * 60 + 26 * 60 + 34) * 1000000000L
+                + 437582643)); // TIME(9), 14:26:34.437582643
+    row.put(
+        "epochsec",
+        String.valueOf(
+            Math.abs(r.nextLong()) % 1638459438)); // TIMESTAMP_LTZ(0), 02.12.2021 15:37:18
+    row.put(
+        "epochnano",
+        String.format(
+            "%d%d",
+            Math.abs(r.nextInt()) % 1999999999,
+            100000000
+                + Math.abs(
+                    r.nextInt(899999999)))); // TIMESTAMP_LTZ(9), 18.05.2033 03:33:19.999999999
+    return row;
   }
 
-  private Map<String, Object> getPosRow() {
-    Map<String, Object> posRow = new HashMap<>();
-    posRow.put("num", 123);
-    posRow.put("num_10_5", new BigDecimal("12345.54321"));
-    posRow.put("num_38_0", new BigDecimal("91234567899876543219876543211234567891"));
-    posRow.put("num_2_0", 12);
-    posRow.put("num_4_0", 1234);
-    posRow.put("num_9_0", 123456789);
-    posRow.put("num_18_0", 123456789987654321L);
-    posRow.put("num_float", 12345.54321f);
-    posRow.put("str_varchar", "abcd123456.");
-    posRow.put("bin", new byte[] {4, 5, 6});
-    posRow.put("bl", true);
-    posRow.put(
-        "var",
-        "{ \"a\": 1, \"b\": \"qwerty\", \"c\": null, \"d\": { \"e\": 2, \"f\": \"asdf\", \"g\":"
-            + " null } }");
-    posRow.put(
-        "obj",
-        "{ \"a\": 1, \"b\": \"qwerty\", \"c\": null, \"d\": { \"e\": 2, \"f\": \"asdf\", \"g\":"
-            + " null } }");
-    posRow.put("arr", Arrays.asList("{ \"a\": 1}", "{ \"b\": 2 }", "{ \"c\": 3 }"));
-    posRow.put("epochdays", "2022-09-18 20:05:07"); // DATE, 18.09.2022
-    posRow.put("epochsec", "2022-09-18 20:05:07"); // TIMESTAMP_NTZ(0)
-    posRow.put("epochnano", "2022-09-18 20:05:07.999999999"); // TIMESTAMP_NTZ(9)
-    posRow.put("timesec", "01:00:01.999999999"); // TIME(0)
-    posRow.put("timenano", "01:00:01.999999999"); // TIME(9)
-
-    return posRow;
+  private static long nextLongOfPrecision(Random r, int precision) {
+    return r.nextLong() % Math.round(Math.pow(10, precision));
   }
 
-  private void assertNegRow(Map<String, Object> expectedNegRow, ResultSet actualResult)
-      throws SQLException {
-    assertNonTimeAndVarFields(expectedNegRow, actualResult);
-    Assert.assertEquals("{}", actualResult.getString("VAR"));
-    Assert.assertEquals("{}", actualResult.getString("OBJ"));
-    Assert.assertEquals("[]", actualResult.getString("ARR"));
-    Assert.assertEquals(0, actualResult.getDate("EPOCHDAYS").getTime());
-    Assert.assertEquals(0, actualResult.getTimestamp("EPOCHSEC").getTime());
-    Assert.assertEquals(0, actualResult.getTimestamp("EPOCHNANO").getNanos());
-    Assert.assertEquals(0, actualResult.getTimestamp("EPOCHNANO").getTime());
-    Assert.assertEquals(0, actualResult.getTimestamp("TIMESEC").getTime());
-    Assert.assertEquals(0, actualResult.getTimestamp("TIMENANO").getNanos());
-    Assert.assertEquals(0, actualResult.getTimestamp("TIMENANO").getTime());
+  private static String nextString(Random r) {
+    return new String(nextBytes(r));
   }
 
-  private void assertNullRow(Map<String, Object> expectedNullRow, ResultSet actualResult)
-      throws SQLException {
-    Assert.assertNotNull(actualResult);
-    assertNonTimeAndVarFields(expectedNullRow, actualResult);
-    Assert.assertEquals(null, actualResult.getString("VAR"));
-    Assert.assertEquals(null, actualResult.getString("OBJ"));
-    Assert.assertEquals(null, actualResult.getString("ARR"));
-    Assert.assertEquals(null, actualResult.getDate("EPOCHDAYS"));
-    Assert.assertEquals(null, actualResult.getTimestamp("EPOCHSEC"));
-    Assert.assertEquals(null, actualResult.getTimestamp("EPOCHNANO"));
-    Assert.assertEquals(null, actualResult.getTimestamp("TIMESEC"));
-    Assert.assertEquals(null, actualResult.getTimestamp("TIMENANO"));
+  private static byte[] nextBytes(Random r) {
+    byte[] bin = new byte[128];
+    r.nextBytes(bin);
+    for (int i = 0; i < bin.length; i++) {
+      bin[i] = (byte) (Math.abs(bin[i]) % 25 + 97); // ascii letters
+    }
+    return bin;
   }
 
-  private void assertPosRow(Map<String, Object> expectedPosRow, ResultSet actualResult)
-      throws SQLException {
-    String formattedJSON =
-        "{\n"
-            + "  \"a\": 1,\n"
-            + "  \"b\": \"qwerty\",\n"
-            + "  \"c\": null,\n"
-            + "  \"d\": {\n"
-            + "    \"e\": 2,\n"
-            + "    \"f\": \"asdf\",\n"
-            + "    \"g\": null\n"
-            + "  }\n"
-            + "}";
-
-    String formattedArray =
-        "[\n"
-            + "  \"{ \\\"a\\\": 1}\",\n"
-            + "  \"{ \\\"b\\\": 2 }\",\n"
-            + "  \"{ \\\"c\\\": 3 }\"\n"
-            + "]";
-    Assert.assertNotNull(actualResult);
-    assertNonTimeAndVarFields(expectedPosRow, actualResult);
-    Assert.assertEquals(formattedJSON, actualResult.getString("VAR"));
-    Assert.assertEquals(formattedJSON, actualResult.getString("OBJ"));
-    Assert.assertEquals(formattedArray, actualResult.getString("ARR"));
-    Assert.assertEquals(
-        1663459200000l, actualResult.getDate("EPOCHDAYS").getTime()); // in ms, 18.09.2022 00:00:00
-    Assert.assertEquals(1663531507000L, actualResult.getTimestamp("EPOCHSEC").getTime());
-    Assert.assertEquals(999999999L, actualResult.getTimestamp("EPOCHNANO").getNanos());
-    // 1663531507000 ms + 999 ms (from ns)
-    Assert.assertEquals(1663531507999L, actualResult.getTimestamp("EPOCHNANO").getTime());
-    Assert.assertEquals(3601000, actualResult.getTimestamp("TIMESEC").getTime()); // 1h + 1s in ms
-    Assert.assertEquals(999999999, actualResult.getTimestamp("TIMENANO").getNanos());
-    // 1h + 1s + 999 ms (from ns)
-    Assert.assertEquals(3601999, actualResult.getTimestamp("TIMENANO").getTime());
+  private static double nextFloat(Random r) {
+    return (r.nextLong() % Math.round(Math.pow(10, 10))) / 100000d;
   }
 
-  private void assertNonTimeAndVarFields(Map<String, Object> row, ResultSet result)
-      throws SQLException {
-    Assert.assertNotNull(result);
-    Assert.assertEquals(row.get("num"), result.getInt("NUM"));
-    Assert.assertEquals(row.get("num_10_5"), result.getBigDecimal("NUM_10_5"));
-    Assert.assertEquals(row.get("num_38_0"), result.getBigDecimal("NUM_38_0"));
-    Assert.assertEquals(row.get("num_2_0"), result.getInt("NUM_2_0"));
-    Assert.assertEquals(row.get("num_4_0"), result.getInt("NUM_4_0"));
-    Assert.assertEquals(row.get("num_9_0"), result.getInt("NUM_9_0"));
-    Assert.assertEquals((long) row.get("num_18_0"), result.getLong("NUM_18_0"));
-    Assert.assertEquals((float) row.get("num_float"), result.getFloat("NUM_FLOAT"), 0);
-    Assert.assertEquals(row.get("str_varchar"), result.getString("STR_VARCHAR"));
-    Assert.assertArrayEquals((byte[]) row.get("bin"), result.getBytes("BIN"));
-    Assert.assertEquals(row.get("bl"), result.getBoolean("BL"));
+  private static String nextJson(Random r) {
+    return String.format(
+        "{ \"%s\": %d, \"%s\": \"%s\", \"%s\": null, \"%s\": { \"%s\": %f, \"%s\": \"%s\", \"%s\":"
+            + " null } }",
+        nextString(r),
+        r.nextInt(),
+        nextString(r),
+        nextString(r),
+        nextString(r),
+        nextString(r),
+        nextString(r),
+        r.nextFloat(),
+        nextString(r),
+        nextString(r),
+        nextString(r));
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index 31afda1f0..0ea0e5de4 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -9,8 +9,6 @@
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.Arrays;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
@@ -21,27 +19,12 @@
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
-import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.SFException;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 
-@RunWith(Parameterized.class)
 public abstract class AbstractDataTypeTest {
-  @Parameterized.Parameters(name = "{0}")
-  public static Collection<Object[]> bdecVersion() {
-    return Arrays.asList(
-        new Object[][] {
-          {"Arrow", Constants.BdecVersion.ONE},
-          // TODO: uncomment once SNOW-659721 is deployed and we set the parameter
-          // DISABLE_PARQUET_CACHE to true for the test account
-          // {"Parquet", Constants.BdecVersion.THREE}
-        });
-  }
-
   private static final String SOURCE_COLUMN_NAME = "source";
   private static final String VALUE_COLUMN_NAME = "value";
 
@@ -69,13 +52,6 @@ protected String randomString() {
 
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
-  private final Constants.BdecVersion bdecVersion;
-
-  public AbstractDataTypeTest(
-      @SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
-    this.bdecVersion = bdecVersion;
-  }
-
   @Before
   public void before() throws Exception {
     databaseName = String.format("SDK_DATATYPE_COMPATIBILITY_IT_%s", randomString());
@@ -89,13 +65,7 @@ public void before() throws Exception {
 
     conn.createStatement().execute(String.format("use warehouse %s;", TestUtils.getWarehouse()));
 
-    if (bdecVersion == Constants.BdecVersion.THREE) {
-      // TODO: encryption and interleaved mode are not yet supported by server side's Parquet
-      // scanner if local file cache is enabled (SNOW-656500)
-      conn.createStatement().execute("alter session set disable_parquet_cache=true;");
-    }
-
-    Properties props = TestUtils.getProperties(bdecVersion);
+    Properties props = TestUtils.getProperties();
     if (props.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       props.setProperty(ROLE, "ACCOUNTADMIN");
     }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
index a7b9d6cd7..f0b0d8199 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
@@ -1,15 +1,10 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Ignore;
 import org.junit.Test;
 
 public class BinaryIT extends AbstractDataTypeTest {
 
-  public BinaryIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testBinarySimple() throws Exception {
     testJdbcTypeCompatibility("BINARY", new byte[0], new ByteArrayProvider());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
index 3a9445ea2..bad5eac16 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
@@ -8,7 +8,6 @@
 import java.time.OffsetTime;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Ignore;
 import org.junit.Test;
 
@@ -18,10 +17,6 @@
  */
 public class DateTimeIT extends AbstractDataTypeTest {
 
-  public DateTimeIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testTimestampWithTimeZone() throws Exception {
     useLosAngelesTimeZone();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
index 48936b950..1fdd2ad5e 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
@@ -2,15 +2,9 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class LogicalTypesIT extends AbstractDataTypeTest {
-
-  public LogicalTypesIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testLogicalTypes() throws Exception {
     // Test booleans
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
index 22c59d3ea..57867427d 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
@@ -1,15 +1,9 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
 import java.util.Arrays;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class NullIT extends AbstractDataTypeTest {
-
-  public NullIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testNullIngestion() throws Exception {
     for (String type :
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
index 64c9bc3fc..1b1c8d280 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
@@ -2,15 +2,9 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class NumericTypesIT extends AbstractDataTypeTest {
-
-  public NumericTypesIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testIntegers() throws Exception {
     // test bytes
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
index 1195d2bfc..3c4ddde22 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
@@ -15,7 +15,6 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -25,10 +24,6 @@ public class SemiStructuredIT extends AbstractDataTypeTest {
   // server-side representation. Validation leaves a small buffer for this difference.
   private static final int MAX_ALLOWED_LENGTH = 16 * 1024 * 1024 - 64;
 
-  public SemiStructuredIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testVariant() throws Exception {
     // Test dates
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
index 198d1c99c..4bcf39519 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
@@ -2,16 +2,10 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Ignore;
 import org.junit.Test;
 
 public class StringsIT extends AbstractDataTypeTest {
-
-  public StringsIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testStrings() throws Exception {
     testJdbcTypeCompatibility("VARCHAR", "", new StringProvider());

From 54fe3c7ca6a7202bc02cd9629f8ef414b771116c Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Wed, 19 Oct 2022 15:26:14 +0200
Subject: [PATCH 080/356] Introduce parquet file generator  (#251)

@snow SNOW-633432 Parquet file generator (#200)
Co-authored-by: Gloria Doci, Andrey Zagrebin
---
 pom.xml                                       |  22 +
 scripts/check_content.sh                      |  21 +-
 .../streaming/internal/ArrowRowBuffer.java    |  10 +-
 .../streaming/internal/BlobMetadata.java      |  17 +-
 .../internal/BlobMetadataWithBdecVersion.java |  25 +
 .../streaming/internal/ChannelCache.java      |   3 +-
 .../internal/DataValidationUtil.java          |  11 +
 .../streaming/internal/FlushService.java      |   7 +-
 .../ingest/streaming/internal/Flusher.java    |   3 +-
 .../streaming/internal/ParquetChunkData.java  |  25 +
 .../streaming/internal/ParquetFlusher.java    | 348 +++++++++++
 .../streaming/internal/ParquetRowBuffer.java  | 222 +++++++
 .../internal/ParquetTypeGenerator.java        | 292 +++++++++
 .../internal/ParquetValueParser.java          | 323 ++++++++++
 .../streaming/internal/RegisterService.java   |   3 +-
 .../ingest/streaming/internal/RowBuffer.java  |   3 +-
 ...owflakeStreamingIngestChannelInternal.java |  20 +-
 ...nowflakeStreamingIngestClientInternal.java |   5 +-
 .../net/snowflake/ingest/utils/Constants.java |   9 +-
 .../java/net/snowflake/ingest/TestUtils.java  |  12 +-
 .../streaming/internal/ArrowBufferTest.java   | 242 ++++----
 .../internal/ColumnMetadataBuilder.java       | 164 +++++
 .../streaming/internal/FlushServiceTest.java  |  50 +-
 .../internal/ParquetTypeGeneratorTest.java    | 480 +++++++++++++++
 .../internal/ParquetValueParserTest.java      | 570 ++++++++++++++++++
 .../streaming/internal/RowBufferTest.java     |   7 +-
 .../streaming/internal/StreamingIngestIT.java | 289 ++++++---
 .../datatypes/AbstractDataTypeTest.java       |  32 +-
 .../internal/datatypes/BinaryIT.java          |   5 +
 .../internal/datatypes/DateTimeIT.java        |   5 +
 .../internal/datatypes/LogicalTypesIT.java    |   6 +
 .../streaming/internal/datatypes/NullIT.java  |   6 +
 .../internal/datatypes/NumericTypesIT.java    |   6 +
 .../internal/datatypes/SemiStructuredIT.java  |   5 +
 .../internal/datatypes/StringsIT.java         |   6 +
 35 files changed, 2992 insertions(+), 262 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java

diff --git a/pom.xml b/pom.xml
index cb6b2eda0..23726b1d4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -35,6 +35,13 @@
       <url>https://github.com/snowflakedb/snowflake-ingest-java/tree/master</url>
     </scm>
 
+    <repositories>
+        <repository>
+            <id>cloudera-repo</id>
+            <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
+        </repository>
+    </repositories>
+
     <!-- Set our Language Level to Java 8 -->
     <properties>
         <maven.compiler.source>1.8</maven.compiler.source>
@@ -43,6 +50,7 @@
         <jacoco.skip.instrument>true</jacoco.skip.instrument>
         <jacoco.version>0.8.5</jacoco.version>
         <arrow.version>9.0.0</arrow.version>
+        <hadoop.version>2.10.1</hadoop.version>
     </properties>
 
     <build>
@@ -292,6 +300,20 @@
             <version>${arrow.version}</version>
         </dependency>
 
+        <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop -->
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-hadoop</artifactId>
+            <version>1.10.99.7.2.15.0-147</version>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>3.1.1.7.2.15.0-147</version>
+        </dependency>
+
         <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-core -->
         <dependency>
             <groupId>io.dropwizard.metrics</groupId>
diff --git a/scripts/check_content.sh b/scripts/check_content.sh
index 9e5b1c404..e4cfe7f2c 100755
--- a/scripts/check_content.sh
+++ b/scripts/check_content.sh
@@ -5,7 +5,26 @@
 set -o pipefail
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-if jar tvf $DIR/../target/snowflake-ingest-sdk.jar  | awk '{print $8}' | grep -v -E "^(net|com)/snowflake" | grep -v -E "(com|net)/\$" | grep -v -E "^META-INF" | grep -v -E "^mozilla" | grep -v mime.types | grep -v project.properties | grep -v -E "javax" | grep -v -E "^org/" | grep -v -E "^com/google" | grep -v -E "^com/sun" | grep -v "log4j.properties" | grep -v "git.properties" | grep -v "io/" | grep -v "codegen/" | grep -v "com/codahale/" | grep -v "com/ibm/" | grep -v "LICENSE" | grep -v "aix/" | grep -v "darwin/" | grep -v "win/" | grep -v "freebsd/" | grep -v "linux/" | grep -v "com/github/"; then
+if jar tvf $DIR/../target/snowflake-ingest-sdk.jar  | awk '{print $8}' | grep -v -E "^(net|com)/snowflake" \
+    | grep -v -E "(com|net)/\$" | grep -v -E "^META-INF" | grep -v -E "^mozilla" | grep -v mime.types \
+    | grep -v project.properties | grep -v -E "javax" | grep -v -E "^org/" | grep -v -E "^com/google" \
+    | grep -v -E "^com/sun" | grep -v "log4j.properties" | grep -v "git.properties" | grep -v "io/" \
+    | grep -v "codegen/" | grep -v "com/codahale/" | grep -v "com/ibm/" | grep -v "LICENSE" | grep -v "aix/" \
+    | grep -v "darwin/" | grep -v "win/" | grep -v "freebsd/" | grep -v "linux/" | grep -v "com/github/" \
+    | grep -v -E "shaded/" | grep -v "webapps/"  | grep -v "microsoft/" | grep -v "com/ctc/" | grep -v "jersey/" \
+    | grep -v "edu/" | grep -v "com/jcraft/" | grep -v "contribs/" | grep -v "com/zaxxer/"  | grep -v "com/squareup/" \
+    | grep -v "com/thoughtworks/" | grep -v "com/jamesmurty/"  | grep -v "net/iharder/" \
+    | grep -v -E "^core-default.xml" \
+    | grep -v "yarn-version-info.properties" | grep -v "yarn-version-info.properties" \
+    | grep -v "about.html" | grep -v "jetty-dir.css" \
+    | grep -v "krb5_udp-template.conf" | grep -v "krb5-template.conf" \
+    | grep -v "ccache.txt" | grep -v "keytab.txt" \
+    | grep -v "common-version-info.properties" | grep -v "LocalizedFormats_fr.properties" \
+    | grep -v "org.apache.hadoop.application-classloader.properties" | grep -v "assets/" | grep -v "ehcache-core.xsd" \
+    | grep -v "ehcache-107ext.xsd" | grep -v "parquet.thrift" | grep -v "mapred-default.xml" \
+    | grep -v "yarn-default.xml" | grep -v ".keep"  | grep -v "NOTICE" | grep -v "digesterRules.xml" \
+    | grep -v "properties.dtd" | grep -v "PropertyList-1.0.dtd"  \
+    ; then
   echo "[ERROR] Ingest SDK jar includes class not under the snowflake namespace"
   exit 1
 fi
\ No newline at end of file
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 38b4583a9..2d7d17a92 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -471,15 +471,7 @@ private float convertRowToArrow(
             // vector.setSafe requires the BigDecimal input scale explicitly match its scale
             inputAsBigDecimal = inputAsBigDecimal.setScale(columnScale, RoundingMode.HALF_UP);
 
-            if (inputAsBigDecimal.abs().compareTo(BigDecimal.TEN.pow(columnPrecision - columnScale))
-                >= 0) {
-              throw new SFException(
-                  ErrorCode.INVALID_ROW,
-                  inputAsBigDecimal,
-                  String.format(
-                      "Number out of representable exclusive range of (-1e%s..1e%s)",
-                      columnPrecision - columnScale, columnPrecision - columnScale));
-            }
+            DataValidationUtil.checkValueInRange(inputAsBigDecimal, columnScale, columnPrecision);
 
             if (columnScale != 0 || physicalType == ColumnPhysicalType.SB16) {
               ((DecimalVector) vector).setSafe(curRowIndex, inputAsBigDecimal);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index 2818a392a..f90ba5055 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -49,9 +49,16 @@ List<ChunkMetadata> getChunks() {
     return this.chunks;
   }
 
-  // TODO: send the bdec_version once server side supports this in production
-  //  @JsonProperty("bdec_version")
-  //  byte getVersionByte() {
-  //    return bdecVersion.toByte();
-  //  }
+  /**
+   * Create {@link BlobMetadata} in case of {@link Constants.BdecVersion#ONE} and {@link
+   * BlobMetadataWithBdecVersion} otherwise to send BDEC version to server side.
+   */
+  static BlobMetadata createBlobMetadata(
+      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks) {
+    // TODO SNOW-659721: Unify BlobMetadata with BlobMetadataWithBdecVersion once server side
+    // BdecVersion in production
+    return bdecVersion == Constants.BdecVersion.ONE
+        ? new BlobMetadata(path, md5, bdecVersion, chunks)
+        : new BlobMetadataWithBdecVersion(path, md5, bdecVersion, chunks);
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
new file mode 100644
index 000000000..7f9c602e3
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+import net.snowflake.ingest.utils.Constants;
+
+/**
+ * Same as {@link BlobMetadata} but with {@link Constants.BdecVersion} to send it to server side if
+ * the version is greater than {@link Constants.BdecVersion#ONE}.
+ */
+class BlobMetadataWithBdecVersion extends BlobMetadata {
+  BlobMetadataWithBdecVersion(
+      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks) {
+    super(path, md5, bdecVersion, chunks);
+  }
+
+  @JsonProperty("bdec_version")
+  byte getVersionByte() {
+    return getVersion().toByte();
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
index 35ca580bc..ef366b120 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
@@ -14,7 +14,8 @@
  * during flush. The key is a fully qualified table name and the value is a set of channels that
  * belongs to this table
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 class ChannelCache<T> {
   // Cache to hold all the valid channels, the key for the outer map is FullyQualifiedTableName and
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 881a4c2f3..de7185009 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -674,6 +674,17 @@ static int validateAndParseBoolean(Object input) {
         input.getClass(), "BOOLEAN", new String[] {"boolean", "Number", "String"});
   }
 
+  static void checkValueInRange(BigDecimal bigDecimalValue, int scale, int precision) {
+    if (bigDecimalValue.abs().compareTo(BigDecimal.TEN.pow(precision - scale)) >= 0) {
+      throw new SFException(
+          ErrorCode.INVALID_ROW,
+          bigDecimalValue,
+          String.format(
+              "Number out of representable exclusive range of (-1e%s..1e%s)",
+              precision - scale, precision - scale));
+    }
+  }
+
   static Set<String> allowedBooleanStringsLowerCased =
       Sets.newHashSet("1", "0", "yes", "no", "y", "n", "t", "f", "true", "false", "on", "off");
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index df5522469..0ef790c6f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -58,7 +58,8 @@
  * <li>upload the blob to stage
  * <li>register the blob to the targeted Snowflake table
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 class FlushService<T> {
 
@@ -467,7 +468,6 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
         // We need to maintain IV as a block counter for the whole file, even interleaved,
         // to align with decryption on the Snowflake query path.
         // TODO: address alignment for the header SNOW-557866
-        // TODO: encryption is not yet supported by server side for Parquet
         long iv = curDataSize / Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES;
         byte[] encryptedCompressedChunkData =
             Cryptor.encrypt(
@@ -559,7 +559,8 @@ BlobMetadata upload(String filePath, byte[] blob, List<ChunkMetadata> metadata)
         blob.length,
         System.currentTimeMillis() - startTime);
 
-    return new BlobMetadata(filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata);
+    return BlobMetadata.createBlobMetadata(
+        filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
index 4ac7acf8c..37ab4afc4 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
@@ -13,7 +13,8 @@
  * Interface to convert {@link ChannelData} buffered in {@link RowBuffer} to the underlying format
  * implementation for faster processing.
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 public interface Flusher<T> {
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
new file mode 100644
index 000000000..0c7b12ff2
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.util.List;
+import java.util.Map;
+
+/** Parquet data holder to buffer rows. */
+public class ParquetChunkData {
+  final List<List<Object>> rows;
+  final Map<String, String> metadata;
+
+  /**
+   * Construct parquet data chunk.
+   *
+   * @param rows chunk row set
+   * @param metadata chunk metadata
+   */
+  public ParquetChunkData(List<List<Object>> rows, Map<String, String> metadata) {
+    this.rows = rows;
+    this.metadata = metadata;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
new file mode 100644
index 000000000..3d9827b6b
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
@@ -0,0 +1,348 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import static net.snowflake.ingest.utils.Constants.MAX_CHUNK_SIZE_IN_BYTES;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.ParquetProperties;
+import org.apache.parquet.hadoop.ParquetFileWriter;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.api.WriteSupport;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.io.DelegatingPositionOutputStream;
+import org.apache.parquet.io.OutputFile;
+import org.apache.parquet.io.ParquetEncodingException;
+import org.apache.parquet.io.PositionOutputStream;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.api.RecordConsumer;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+
+/**
+ * Converts {@link ChannelData} buffered in {@link RowBuffer} to the Parquet format for faster
+ * processing.
+ */
+public class ParquetFlusher implements Flusher<ParquetChunkData> {
+  private static final Logging logger = new Logging(ParquetFlusher.class);
+  private final MessageType schema;
+
+  /** Construct parquet flusher from its schema. */
+  public ParquetFlusher(MessageType schema) {
+    this.schema = schema;
+  }
+
+  @Override
+  public SerializationResult serialize(
+      List<ChannelData<ParquetChunkData>> channelsDataPerTable,
+      ByteArrayOutputStream chunkData,
+      String filePath)
+      throws IOException {
+    List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
+    long rowCount = 0L;
+    SnowflakeStreamingIngestChannelInternal<ParquetChunkData> firstChannel = null;
+    Map<String, RowBufferStats> columnEpStatsMapCombined = null;
+    List<List<Object>> rows = null;
+
+    for (ChannelData<ParquetChunkData> data : channelsDataPerTable) {
+      // Create channel metadata
+      ChannelMetadata channelMetadata =
+          ChannelMetadata.builder()
+              .setOwningChannel(data.getChannel())
+              .setRowSequencer(data.getRowSequencer())
+              .setOffsetToken(data.getOffsetToken())
+              .build();
+      // Add channel metadata to the metadata list
+      channelsMetadataList.add(channelMetadata);
+
+      logger.logDebug(
+          "Parquet Flusher: Start building channel={}, rowCount={}, bufferSize={} in blob={}",
+          data.getChannel().getFullyQualifiedName(),
+          data.getRowCount(),
+          data.getBufferSize(),
+          filePath);
+
+      if (rows == null) {
+        columnEpStatsMapCombined = data.getColumnEps();
+        rows = new ArrayList<>();
+        firstChannel = data.getChannel();
+      } else {
+        // This method assumes that channelsDataPerTable is grouped by table. We double check
+        // here and throw an error if the assumption is violated
+        if (!data.getChannel()
+            .getFullyQualifiedTableName()
+            .equals(firstChannel.getFullyQualifiedTableName())) {
+          throw new SFException(ErrorCode.INVALID_DATA_IN_CHUNK);
+        }
+
+        columnEpStatsMapCombined =
+            ChannelData.getCombinedColumnStatsMap(columnEpStatsMapCombined, data.getColumnEps());
+      }
+      rows.addAll(data.getVectors().rows);
+
+      rowCount += data.getRowCount();
+
+      logger.logDebug(
+          "Parquet Flusher: Finish building channel={}, rowCount={}, bufferSize={} in blob={}",
+          data.getChannel().getFullyQualifiedName(),
+          data.getRowCount(),
+          data.getBufferSize(),
+          filePath);
+    }
+
+    Map<String, String> metadata = channelsDataPerTable.get(0).getVectors().metadata;
+
+    flushToParquetBdecChunk(chunkData, rows, metadata, channelsMetadataList);
+    return new SerializationResult(channelsMetadataList, columnEpStatsMapCombined, rowCount);
+  }
+
+  /**
+   * Flushes a parquet row chunk to the given BDEC output stream.
+   *
+   * @param bdecOutput BDEC output stream
+   * @param chunkRows chunk rows
+   * @param metadata chunk metadata
+   * @param channelsMetadataList metadata of the channels the chunk rows belong to
+   * @throws IOException thrown from Parquet library in case of writing problems
+   */
+  private void flushToParquetBdecChunk(
+      ByteArrayOutputStream bdecOutput,
+      List<List<Object>> chunkRows,
+      Map<String, String> metadata,
+      List<ChannelMetadata> channelsMetadataList)
+      throws IOException {
+    try {
+      ParquetWriter<List<Object>> writer =
+          new BdecParquetWriterBuilder(bdecOutput, schema, metadata, channelsMetadataList)
+              // PARQUET_2_0 uses Encoding.DELTA_BYTE_ARRAY for byte arrays (e.g. SF sb16)
+              // server side does not support it TODO: SNOW-657238
+              .withWriterVersion(ParquetProperties.WriterVersion.PARQUET_1_0)
+
+              // the dictionary encoding (Encoding.*_DICTIONARY) is not supported by server side
+              // scanner yet
+              .withDictionaryEncoding(false)
+
+              // Historically server side scanner supports only the case when the row number in all
+              // pages is the same.
+              // The quick fix is to effectively disable the page size/row limit
+              // to always have one page per chunk until server side is generalised.
+              .withPageSize((int) Constants.MAX_CHUNK_SIZE_IN_BYTES * 2)
+              .withPageRowCountLimit(chunkRows.size() + 1)
+              .enableValidation()
+              .withCompressionCodec(CompressionCodecName.GZIP)
+              .withWriteMode(ParquetFileWriter.Mode.CREATE)
+              .build();
+
+      // We can use lower level column writers and custom ValuesWriterFactory that uses plain byte
+      // array encoding used by PARQUET_1_0 and supported by server side
+      // TODO: SNOW-672143
+      for (List<Object> row : chunkRows) {
+        writer.write(row);
+      }
+      writer.close();
+    } catch (Throwable t) {
+      logger.logError("Parquet Flusher: failed to write", t);
+      throw t;
+    }
+  }
+
+  /**
+   * A parquet specific write builder.
+   *
+   * <p>This class is implemented as parquet library API requires, mostly to provide {@link
+   * BdecWriteSupport} implementation.
+   */
+  private static class BdecParquetWriterBuilder
+      extends ParquetWriter.Builder<List<Object>, BdecParquetWriterBuilder> {
+    private final MessageType schema;
+    private final Map<String, String> extraMetaData;
+    private final List<ChannelMetadata> channelsMetadataList;
+
+    protected BdecParquetWriterBuilder(
+        ByteArrayOutputStream stream,
+        MessageType schema,
+        Map<String, String> extraMetaData,
+        List<ChannelMetadata> channelsMetadataList) {
+      super(new ByteArrayOutputFile(stream));
+      this.schema = schema;
+      this.extraMetaData = extraMetaData;
+      this.channelsMetadataList = channelsMetadataList;
+    }
+
+    @Override
+    protected BdecParquetWriterBuilder self() {
+      return this;
+    }
+
+    @Override
+    protected WriteSupport<List<Object>> getWriteSupport(Configuration conf) {
+      return new BdecWriteSupport(schema, extraMetaData, channelsMetadataList);
+    }
+  }
+
+  /**
+   * A parquet specific file output implementation.
+   *
+   * <p>This class is implemented as parquet library API requires, mostly to create our {@link
+   * ByteArrayDelegatingPositionOutputStream} implementation.
+   */
+  private static class ByteArrayOutputFile implements OutputFile {
+    private final ByteArrayOutputStream stream;
+
+    private ByteArrayOutputFile(ByteArrayOutputStream stream) {
+      this.stream = stream;
+    }
+
+    @Override
+    public PositionOutputStream create(long blockSizeHint) throws IOException {
+      stream.reset();
+      return new ByteArrayDelegatingPositionOutputStream(stream);
+    }
+
+    @Override
+    public PositionOutputStream createOrOverwrite(long blockSizeHint) throws IOException {
+      return create(blockSizeHint);
+    }
+
+    @Override
+    public boolean supportsBlockSize() {
+      return false;
+    }
+
+    @Override
+    public long defaultBlockSize() {
+      return (int) MAX_CHUNK_SIZE_IN_BYTES;
+    }
+  }
+
+  /**
+   * A parquet specific output stream implementation.
+   *
+   * <p>This class is implemented as parquet library API requires, mostly to wrap our BDEC output
+   * {@link ByteArrayOutputStream}.
+   */
+  private static class ByteArrayDelegatingPositionOutputStream
+      extends DelegatingPositionOutputStream {
+    private final ByteArrayOutputStream stream;
+
+    public ByteArrayDelegatingPositionOutputStream(ByteArrayOutputStream stream) {
+      super(stream);
+      this.stream = stream;
+    }
+
+    @Override
+    public long getPos() {
+      return stream.size();
+    }
+  }
+
+  /**
+   * A parquet specific write support implementation.
+   *
+   * <p>This class is implemented as parquet library API requires, mostly to serialize user column
+   * values depending on type into Parquet {@link RecordConsumer} in {@link
+   * BdecWriteSupport#write(List)}.
+   */
+  private static class BdecWriteSupport extends WriteSupport<List<Object>> {
+    MessageType schema;
+    RecordConsumer recordConsumer;
+    Map<String, String> extraMetadata;
+    List<ChannelMetadata> channelsMetadataList;
+
+    // TODO SNOW-672156: support specifying encodings and compression
+    BdecWriteSupport(
+        MessageType schema,
+        Map<String, String> extraMetadata,
+        List<ChannelMetadata> channelsMetadataList) {
+      this.schema = schema;
+      this.extraMetadata = extraMetadata;
+      this.channelsMetadataList = channelsMetadataList;
+    }
+
+    @Override
+    public WriteContext init(Configuration config) {
+      return new WriteContext(schema, extraMetadata);
+    }
+
+    @Override
+    public void prepareForWrite(RecordConsumer recordConsumer) {
+      this.recordConsumer = recordConsumer;
+    }
+
+    @Override
+    public void write(List<Object> values) {
+      List<ColumnDescriptor> cols = schema.getColumns();
+      if (values.size() != cols.size()) {
+        List<String> channelNames =
+            this.channelsMetadataList.stream()
+                .map(ChannelMetadata::getChannelName)
+                .collect(Collectors.toList());
+        throw new ParquetEncodingException(
+            "Invalid input data in channels "
+                + channelNames
+                + ". Expecting "
+                + cols.size()
+                + " columns. Input had "
+                + values.size()
+                + " columns ("
+                + cols
+                + ") : "
+                + values);
+      }
+
+      recordConsumer.startMessage();
+      for (int i = 0; i < cols.size(); ++i) {
+        Object val = values.get(i);
+        // val.length() == 0 indicates a NULL value.
+        if (val != null) {
+          String fieldName = cols.get(i).getPath()[0];
+          recordConsumer.startField(fieldName, i);
+          PrimitiveType.PrimitiveTypeName typeName =
+              cols.get(i).getPrimitiveType().getPrimitiveTypeName();
+          switch (typeName) {
+            case BOOLEAN:
+              recordConsumer.addBoolean((boolean) val);
+              break;
+            case FLOAT:
+              recordConsumer.addFloat((float) val);
+              break;
+            case DOUBLE:
+              recordConsumer.addDouble((double) val);
+              break;
+            case INT32:
+              recordConsumer.addInteger((int) val);
+              break;
+            case INT64:
+              recordConsumer.addLong((long) val);
+              break;
+            case BINARY:
+              recordConsumer.addBinary(Binary.fromString((String) val));
+              break;
+            case FIXED_LEN_BYTE_ARRAY:
+              Binary binary = Binary.fromConstantByteArray((byte[]) val);
+              recordConsumer.addBinary(binary);
+              break;
+            default:
+              throw new ParquetEncodingException(
+                  "Unsupported column type: " + cols.get(i).getPrimitiveType());
+          }
+          recordConsumer.endField(fieldName, i);
+        }
+      }
+      recordConsumer.endMessage();
+    }
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
new file mode 100644
index 000000000..de4b3eadc
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -0,0 +1,222 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+
+/**
+ * The buffer in the Streaming Ingest channel that holds the un-flushed rows, these rows will be
+ * converted to Parquet format for faster processing
+ */
+public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
+  private static final Logging logger = new Logging(ParquetRowBuffer.class);
+
+  private static final String PARQUET_MESSAGE_TYPE_NAME = "bdec";
+
+  private final Map<String, Pair<ColumnMetadata, Integer>> fieldIndex;
+  private final Map<String, String> metadata;
+  private final List<List<Object>> data;
+  private final List<List<Object>> tempData;
+
+  private MessageType schema;
+
+  /**
+   * Construct a ParquetRowBuffer object
+   *
+   * @param channel client channel
+   */
+  ParquetRowBuffer(SnowflakeStreamingIngestChannelInternal<ParquetChunkData> channel) {
+    super(channel);
+    fieldIndex = new HashMap<>();
+    metadata = new HashMap<>();
+    data = new ArrayList<>();
+    tempData = new ArrayList<>();
+  }
+
+  @Override
+  public void setupSchema(List<ColumnMetadata> columns) {
+    fieldIndex.clear();
+    metadata.clear();
+    List<Type> parquetTypes = new ArrayList<>();
+    // Snowflake column id that corresponds to the order in 'columns' received from server
+    // id is required to pack column metadata for the server scanner, e.g. decimal scale and
+    // precision
+    int id = 1;
+    for (ColumnMetadata column : columns) {
+      ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+          ParquetTypeGenerator.generateColumnParquetTypeInfo(column, id);
+      parquetTypes.add(typeInfo.getParquetType());
+      this.metadata.putAll(typeInfo.getMetadata());
+      fieldIndex.put(column.getName(), new Pair<>(column, parquetTypes.size() - 1));
+      if (!column.getNullable()) {
+        addNonNullableFieldName(column.getName());
+      }
+      this.statsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
+
+      if (this.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.ABORT) {
+        this.tempStatsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
+      }
+
+      id++;
+    }
+    schema = new MessageType(PARQUET_MESSAGE_TYPE_NAME, parquetTypes);
+  }
+
+  @Override
+  boolean hasColumn(String name) {
+    return fieldIndex.containsKey(name);
+  }
+
+  @Override
+  float addRow(
+      Map<String, Object> row,
+      int curRowIndex,
+      Map<String, RowBufferStats> statsMap,
+      Set<String> formattedInputColumnNames) {
+    return addRow(row, data, statsMap, formattedInputColumnNames);
+  }
+
+  @Override
+  float addTempRow(
+      Map<String, Object> row,
+      int curRowIndex,
+      Map<String, RowBufferStats> statsMap,
+      Set<String> formattedInputColumnNames) {
+    return addRow(row, tempData, statsMap, formattedInputColumnNames);
+  }
+
+  /**
+   * Adds a row to the parquet buffer.
+   *
+   * @param row row to add
+   * @param out internal buffer to add to
+   * @param statsMap column stats map
+   * @param inputColumnNames list of input column names after formatting
+   * @return row size
+   */
+  private float addRow(
+      Map<String, Object> row,
+      List<List<Object>> out,
+      Map<String, RowBufferStats> statsMap,
+      Set<String> inputColumnNames) {
+    Object[] indexedRow = new Object[fieldIndex.size()];
+    float size = 0F;
+    for (Map.Entry<String, Object> entry : row.entrySet()) {
+      String key = entry.getKey();
+      Object value = entry.getValue();
+      String columnName = formatColumnName(key);
+      int colIndex = fieldIndex.get(columnName).getSecond();
+      RowBufferStats stats = statsMap.get(columnName);
+      ColumnMetadata column = fieldIndex.get(columnName).getFirst();
+      ColumnDescriptor columnDescriptor = schema.getColumns().get(colIndex);
+      PrimitiveType.PrimitiveTypeName typeName =
+          columnDescriptor.getPrimitiveType().getPrimitiveTypeName();
+      ParquetValueParser.ParquetBufferValue valueWithSize =
+          ParquetValueParser.parseColumnValueToParquet(value, column, typeName, stats);
+      indexedRow[colIndex] = valueWithSize.getValue();
+      size += valueWithSize.getSize();
+    }
+    out.add(Arrays.asList(indexedRow));
+
+    for (String columnName : Sets.difference(this.fieldIndex.keySet(), inputColumnNames)) {
+      statsMap.get(columnName).incCurrentNullCount();
+    }
+    return size;
+  }
+
+  @Override
+  void moveTempRowsToActualBuffer(int tempRowCount) {
+    data.addAll(tempData);
+  }
+
+  @Override
+  void clearTempRows() {
+    tempData.clear();
+  }
+
+  @Override
+  boolean hasColumns() {
+    return !fieldIndex.isEmpty();
+  }
+
+  @Override
+  Optional<ParquetChunkData> getSnapshot() {
+    List<List<Object>> oldData = new ArrayList<>();
+    data.forEach(r -> oldData.add(new ArrayList<>(r)));
+    return oldData.isEmpty()
+        ? Optional.empty()
+        : Optional.of(new ParquetChunkData(oldData, metadata));
+  }
+
+  /** Used only for testing. */
+  @Override
+  Object getVectorValueAt(String column, int index) {
+    int colIndex = fieldIndex.get(column).getSecond();
+    Object value = data.get(index).get(colIndex);
+    ColumnMetadata columnMetadata = fieldIndex.get(column).getFirst();
+    String physicalTypeStr = columnMetadata.getPhysicalType();
+    ColumnPhysicalType physicalType = ColumnPhysicalType.valueOf(physicalTypeStr);
+    String logicalTypeStr = columnMetadata.getLogicalType();
+    ColumnLogicalType logicalType = ColumnLogicalType.valueOf(logicalTypeStr);
+    if (logicalType == ColumnLogicalType.FIXED) {
+      if (physicalType == ColumnPhysicalType.SB1) {
+        value = ((Integer) value).byteValue();
+      }
+      if (physicalType == ColumnPhysicalType.SB2) {
+        value = ((Integer) value).shortValue();
+      }
+      if (physicalType == ColumnPhysicalType.SB16) {
+        value = new BigDecimal(new BigInteger((byte[]) value), columnMetadata.getScale());
+      }
+    }
+    if (logicalType == ColumnLogicalType.BINARY && value != null) {
+      value = ((String) value).getBytes(StandardCharsets.UTF_8);
+    }
+    return value;
+  }
+
+  @Override
+  int getTempRowCount() {
+    return tempData.size();
+  }
+
+  @Override
+  void reset() {
+    super.reset();
+    data.clear();
+  }
+
+  @Override
+  public void close(String name) {
+    this.fieldIndex.clear();
+    logger.logInfo(
+        "Trying to close parquet buffer for channel={} from function={}",
+        this.owningChannel.getName(),
+        name);
+  }
+
+  @Override
+  public Flusher<ParquetChunkData> createFlusher(Constants.BdecVersion bdecVerion) {
+    return new ParquetFlusher(schema);
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
new file mode 100644
index 000000000..149ce766c
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
@@ -0,0 +1,292 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+import org.apache.parquet.schema.Types;
+
+/** Generates the Parquet types for the Snowflake's column types */
+public class ParquetTypeGenerator {
+
+  /**
+   * Util class that contains Parquet type and other metadata for that type needed by the Snowflake
+   * server side scanner
+   */
+  static class ParquetTypeInfo {
+    private Type parquetType;
+    private Map<String, String> metadata;
+
+    public Type getParquetType() {
+      return this.parquetType;
+    }
+
+    public Map<String, String> getMetadata() {
+      return this.metadata;
+    }
+
+    public void setParquetType(Type parquetType) {
+      this.parquetType = parquetType;
+    }
+
+    public void setMetadata(Map<String, String> metadata) {
+      this.metadata = metadata;
+    }
+  }
+
+  private static final Set<AbstractRowBuffer.ColumnPhysicalType> TIME_SUPPORTED_PHYSICAL_TYPES =
+      new HashSet<>(
+          Arrays.asList(
+              AbstractRowBuffer.ColumnPhysicalType.SB4, AbstractRowBuffer.ColumnPhysicalType.SB8));
+  private static final Set<AbstractRowBuffer.ColumnPhysicalType>
+      TIMESTAMP_SUPPORTED_PHYSICAL_TYPES =
+          new HashSet<>(
+              Arrays.asList(
+                  AbstractRowBuffer.ColumnPhysicalType.SB8,
+                  AbstractRowBuffer.ColumnPhysicalType.SB16));
+
+  /**
+   * Generate the column parquet type and metadata from the column metadata received from server
+   * side.
+   *
+   * @param column column metadata as received from server side
+   * @param id column id
+   * @return column parquet type
+   */
+  static ParquetTypeInfo generateColumnParquetTypeInfo(ColumnMetadata column, int id) {
+    ParquetTypeInfo res = new ParquetTypeInfo();
+    Type parquetType;
+    Map<String, String> metadata = new HashMap<>();
+    String name = column.getName();
+
+    AbstractRowBuffer.ColumnPhysicalType physicalType;
+    AbstractRowBuffer.ColumnLogicalType logicalType;
+    try {
+      physicalType = AbstractRowBuffer.ColumnPhysicalType.valueOf(column.getPhysicalType());
+      logicalType = AbstractRowBuffer.ColumnLogicalType.valueOf(column.getLogicalType());
+    } catch (IllegalArgumentException e) {
+      throw new SFException(
+          ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
+    }
+
+    metadata.put(Integer.toString(id), logicalType.getOrdinal() + "," + physicalType.getOrdinal());
+
+    // Parquet Type.Repetition in general supports repeated values for the same row column, like a
+    // list of values.
+    // This generator uses only either 0 or 1 value for nullable data type (OPTIONAL: 0 or none
+    // value if it is null)
+    // or exactly 1 value for non-nullable data type (REQUIRED)
+    Type.Repetition repetition =
+        column.getNullable() ? Type.Repetition.OPTIONAL : Type.Repetition.REQUIRED;
+
+    // Handle differently depends on the column logical and physical types
+    switch (logicalType) {
+      case FIXED:
+        parquetType = getFixedColumnParquetType(column, id, physicalType, repetition);
+        break;
+      case ARRAY:
+      case OBJECT:
+      case VARIANT:
+        // mark the column metadata as being an object json for the server side scanner
+        metadata.put(id + ":obj_enc", "1");
+        // parquetType is same as the next one
+      case ANY:
+      case CHAR:
+      case TEXT:
+      case BINARY:
+        parquetType =
+            Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition)
+                .as(LogicalTypeAnnotation.stringType())
+                .id(id)
+                .named(name);
+        break;
+      case TIMESTAMP_LTZ:
+      case TIMESTAMP_NTZ:
+      case TIMESTAMP_TZ:
+        parquetType =
+            getTimeColumnParquetType(
+                column.getScale(),
+                physicalType,
+                logicalType,
+                TIMESTAMP_SUPPORTED_PHYSICAL_TYPES,
+                repetition,
+                id,
+                name);
+        break;
+      case DATE:
+        parquetType =
+            Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition)
+                .as(LogicalTypeAnnotation.dateType())
+                .id(id)
+                .named(name);
+        break;
+      case TIME:
+        parquetType =
+            getTimeColumnParquetType(
+                column.getScale(),
+                physicalType,
+                logicalType,
+                TIME_SUPPORTED_PHYSICAL_TYPES,
+                repetition,
+                id,
+                name);
+        break;
+      case BOOLEAN:
+        parquetType =
+            Types.primitive(PrimitiveType.PrimitiveTypeName.BOOLEAN, repetition).id(id).named(name);
+        break;
+      case REAL:
+        parquetType =
+            Types.primitive(PrimitiveType.PrimitiveTypeName.DOUBLE, repetition).id(id).named(name);
+        break;
+      default:
+        throw new SFException(
+            ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
+    }
+    res.setParquetType(parquetType);
+    res.setMetadata(metadata);
+    return res;
+  }
+
+  /**
+   * Get the parquet type for column of Snowflake FIXED logical type.
+   *
+   * @param column column metadata
+   * @param id column id in Snowflake table schema
+   * @param physicalType Snowflake physical type of column
+   * @param repetition parquet repetition type of column
+   * @return column parquet type
+   */
+  private static Type getFixedColumnParquetType(
+      ColumnMetadata column,
+      int id,
+      AbstractRowBuffer.ColumnPhysicalType physicalType,
+      Type.Repetition repetition) {
+    String name = column.getName();
+    // the LogicalTypeAnnotation.DecimalLogicalTypeAnnotation is used by server side scanner
+    // to discover data type scale and precision
+    LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalLogicalTypeAnnotation =
+        column.getScale() != null && column.getPrecision() != null
+            ? LogicalTypeAnnotation.DecimalLogicalTypeAnnotation.decimalType(
+                column.getScale(), column.getPrecision())
+            : null;
+    Type parquetType;
+    if ((column.getScale() != null && column.getScale() != 0)
+        || physicalType == AbstractRowBuffer.ColumnPhysicalType.SB16) {
+      parquetType =
+          Types.primitive(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, repetition)
+              .length(16)
+              .as(decimalLogicalTypeAnnotation)
+              .id(id)
+              .named(name);
+    } else {
+      switch (physicalType) {
+        case SB1:
+        case SB2:
+        case SB4:
+          parquetType =
+              Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition)
+                  .as(decimalLogicalTypeAnnotation)
+                  .id(id)
+                  .length(4)
+                  .named(name);
+          break;
+        case SB8:
+          parquetType =
+              Types.primitive(PrimitiveType.PrimitiveTypeName.INT64, repetition)
+                  .as(decimalLogicalTypeAnnotation)
+                  .id(id)
+                  .length(8)
+                  .named(name);
+          break;
+        default:
+          throw new SFException(
+              ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
+      }
+    }
+    return parquetType;
+  }
+
+  /**
+   * Get the parquet type for column of a Snowflake time logical type.
+   *
+   * @param scale column scale
+   * @param physicalType Snowflake physical type of column
+   * @param logicalType Snowflake logical type of column
+   * @param supportedPhysicalTypes supported Snowflake physical types for the given column
+   * @param repetition parquet repetition type of column
+   * @param id column id in Snowflake table schema
+   * @param name column name
+   * @return column parquet type
+   */
+  private static Type getTimeColumnParquetType(
+      Integer scale,
+      AbstractRowBuffer.ColumnPhysicalType physicalType,
+      AbstractRowBuffer.ColumnLogicalType logicalType,
+      Set<AbstractRowBuffer.ColumnPhysicalType> supportedPhysicalTypes,
+      Type.Repetition repetition,
+      int id,
+      String name) {
+    if (scale == null || scale > 9 || scale < 0 || !supportedPhysicalTypes.contains(physicalType)) {
+      throw new SFException(
+          ErrorCode.UNKNOWN_DATA_TYPE,
+          "Data type: " + logicalType + ", " + physicalType + ", scale: " + scale);
+    }
+
+    PrimitiveType.PrimitiveTypeName type = getTimePrimitiveType(physicalType);
+    LogicalTypeAnnotation typeAnnotation;
+    int length;
+    switch (physicalType) {
+      case SB4:
+        typeAnnotation = LogicalTypeAnnotation.decimalType(scale, 9);
+        length = 4;
+        break;
+      case SB8:
+        typeAnnotation = LogicalTypeAnnotation.decimalType(scale, 18);
+        length = 8;
+        break;
+      case SB16:
+        typeAnnotation = LogicalTypeAnnotation.decimalType(scale, 38);
+        length = 16;
+        break;
+      default:
+        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
+    }
+    return Types.primitive(type, repetition).as(typeAnnotation).length(length).id(id).named(name);
+  }
+
+  /**
+   * Get the parquet primitive type name for column of a Snowflake time logical type.
+   *
+   * @param physicalType Snowflake physical type of column
+   * @return column parquet primitive type name
+   */
+  private static PrimitiveType.PrimitiveTypeName getTimePrimitiveType(
+      AbstractRowBuffer.ColumnPhysicalType physicalType) {
+    PrimitiveType.PrimitiveTypeName type;
+    switch (physicalType) {
+      case SB4:
+        type = PrimitiveType.PrimitiveTypeName.INT32;
+        break;
+      case SB8:
+        type = PrimitiveType.PrimitiveTypeName.INT64;
+        break;
+      case SB16:
+        type = PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY;
+        break;
+      default:
+        throw new UnsupportedOperationException("Time physical type: " + physicalType);
+    }
+    return type;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
new file mode 100644
index 000000000..671af1a5b
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
@@ -0,0 +1,323 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.math.RoundingMode;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import javax.annotation.Nullable;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.parquet.schema.PrimitiveType;
+
+/** Parses a user column value into Parquet internal representation for buffering. */
+class ParquetValueParser {
+  /** Parquet internal value representation for buffering. */
+  static class ParquetBufferValue {
+    private final Object value;
+    private final float size;
+
+    ParquetBufferValue(Object value, float size) {
+      this.value = value;
+      this.size = size;
+    }
+
+    Object getValue() {
+      return value;
+    }
+
+    float getSize() {
+      return size;
+    }
+  }
+
+  /**
+   * Parses a user column value into Parquet internal representation for buffering.
+   *
+   * @param value column value provided by user in a row
+   * @param columnMetadata column metadata
+   * @param typeName Parquet primitive type name
+   * @param stats column stats to update
+   * @return parsed value and byte size of Parquet internal representation
+   */
+  static ParquetBufferValue parseColumnValueToParquet(
+      Object value,
+      ColumnMetadata columnMetadata,
+      PrimitiveType.PrimitiveTypeName typeName,
+      RowBufferStats stats) {
+    Utils.assertNotNull("Parquet column stats", stats);
+    float size = 0F;
+    if (value != null) {
+      AbstractRowBuffer.ColumnLogicalType logicalType =
+          AbstractRowBuffer.ColumnLogicalType.valueOf(columnMetadata.getLogicalType());
+      AbstractRowBuffer.ColumnPhysicalType physicalType =
+          AbstractRowBuffer.ColumnPhysicalType.valueOf(columnMetadata.getPhysicalType());
+      switch (typeName) {
+        case BOOLEAN:
+          int intValue = DataValidationUtil.validateAndParseBoolean(value);
+          value = intValue > 0;
+          stats.addIntValue(BigInteger.valueOf(intValue));
+          size = 1;
+          break;
+        case INT32:
+          int intVal =
+              getInt32Value(
+                  value,
+                  columnMetadata.getScale(),
+                  Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
+                  logicalType,
+                  physicalType);
+          value = intVal;
+          stats.addIntValue(BigInteger.valueOf(intVal));
+          size = 4;
+          break;
+        case INT64:
+          long longValue =
+              getInt64Value(
+                  value,
+                  columnMetadata.getScale(),
+                  Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
+                  logicalType,
+                  physicalType);
+          value = longValue;
+          stats.addIntValue(BigInteger.valueOf(longValue));
+          size = 8;
+          break;
+        case DOUBLE:
+          double doubleValue = DataValidationUtil.validateAndParseReal(value);
+          value = doubleValue;
+          stats.addRealValue(doubleValue);
+          size = 8;
+          break;
+        case BINARY:
+          String str = getBinaryValue(value, stats, columnMetadata);
+          value = str;
+          size = str.getBytes().length;
+          break;
+        case FIXED_LEN_BYTE_ARRAY:
+          BigInteger intRep =
+              getSb16Value(
+                  value,
+                  columnMetadata.getScale(),
+                  Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
+                  logicalType,
+                  physicalType);
+          stats.addIntValue(intRep);
+          value = getSb16Bytes(intRep);
+          size += 16;
+          break;
+        default:
+          throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
+      }
+    } else {
+      if (!columnMetadata.getNullable()) {
+        throw new SFException(
+            ErrorCode.INVALID_ROW, columnMetadata.getName(), "Passed null to non nullable field");
+      }
+      stats.incCurrentNullCount();
+    }
+    return new ParquetBufferValue(value, size);
+  }
+
+  /**
+   * Parses an int32 value based on Snowflake logical type.
+   *
+   * @param value column value provided by user in a row
+   * @param scale data type scale
+   * @param precision data type precision
+   * @param logicalType Snowflake logical type
+   * @param physicalType Snowflake physical type
+   * @return parsed int32 value
+   */
+  private static int getInt32Value(
+      Object value,
+      @Nullable Integer scale,
+      Integer precision,
+      AbstractRowBuffer.ColumnLogicalType logicalType,
+      AbstractRowBuffer.ColumnPhysicalType physicalType) {
+    int intVal;
+    switch (logicalType) {
+      case DATE:
+        intVal = DataValidationUtil.validateAndParseDate(value);
+        break;
+      case TIME:
+        Utils.assertNotNull("Unexpected null scale for TIME data type", scale);
+        intVal = DataValidationUtil.validateAndParseTime(value, scale).intValue();
+        break;
+      case FIXED:
+        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
+        bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
+        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
+        intVal = bigDecimalValue.intValue();
+        break;
+      default:
+        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
+    }
+    return intVal;
+  }
+
+  /**
+   * Parses an int64 value based on Snowflake logical type.
+   *
+   * @param value column value provided by user in a row
+   * @param scale data type scale
+   * @param precision data type precision
+   * @param logicalType Snowflake logical type
+   * @param physicalType Snowflake physical type
+   * @return parsed int64 value
+   */
+  private static long getInt64Value(
+      Object value,
+      int scale,
+      int precision,
+      AbstractRowBuffer.ColumnLogicalType logicalType,
+      AbstractRowBuffer.ColumnPhysicalType physicalType) {
+    long longValue;
+    switch (logicalType) {
+      case TIME:
+        Utils.assertNotNull("Unexpected null scale for TIME data type", scale);
+        longValue = DataValidationUtil.validateAndParseTime(value, scale).longValue();
+        break;
+      case TIMESTAMP_LTZ:
+      case TIMESTAMP_NTZ:
+        boolean ignoreTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
+        longValue =
+            DataValidationUtil.validateAndParseTimestampNtzSb16(value, scale, ignoreTimezone)
+                .getTimeInScale()
+                .longValue();
+        break;
+      case TIMESTAMP_TZ:
+        longValue =
+            DataValidationUtil.validateAndParseTimestampTz(value, scale)
+                .getSfTimestamp()
+                .orElseThrow(
+                    () ->
+                        new SFException(
+                            ErrorCode.INVALID_ROW,
+                            value,
+                            "Unable to parse timestamp for TIMESTAMP_TZ column"))
+                .toBinary(scale, true)
+                .longValue();
+        break;
+      case FIXED:
+        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
+        bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
+        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
+        longValue = bigDecimalValue.longValue();
+        break;
+      default:
+        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
+    }
+    return longValue;
+  }
+
+  /**
+   * Parses an int128 value based on Snowflake logical type.
+   *
+   * @param value column value provided by user in a row
+   * @param scale data type scale
+   * @param precision data type precision
+   * @param logicalType Snowflake logical type
+   * @param physicalType Snowflake physical type
+   * @return parsed int64 value
+   */
+  private static BigInteger getSb16Value(
+      Object value,
+      int scale,
+      int precision,
+      AbstractRowBuffer.ColumnLogicalType logicalType,
+      AbstractRowBuffer.ColumnPhysicalType physicalType) {
+    switch (logicalType) {
+      case TIMESTAMP_TZ:
+        return DataValidationUtil.validateAndParseTimestampTz(value, scale)
+            .getSfTimestamp()
+            .orElseThrow(
+                () ->
+                    new SFException(
+                        ErrorCode.INVALID_ROW,
+                        value,
+                        "Unable to parse timestamp for TIMESTAMP_TZ column"))
+            .toBinary(scale, true);
+      case TIMESTAMP_LTZ:
+      case TIMESTAMP_NTZ:
+        boolean ignoreTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
+        return DataValidationUtil.validateAndParseTimestampNtzSb16(value, scale, ignoreTimezone)
+            .getTimeInScale();
+      case FIXED:
+        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
+        // explicitly match the BigDecimal input scale with the Snowflake data type scale
+        bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
+        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
+        return bigDecimalValue.unscaledValue();
+      default:
+        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
+    }
+  }
+
+  /**
+   * Converts an int128 value to its byte array representation.
+   *
+   * @param intRep int128 value
+   * @return byte array representation
+   */
+  static byte[] getSb16Bytes(BigInteger intRep) {
+    byte[] bytes = intRep.toByteArray();
+    byte padByte = (byte) (bytes[0] < 0 ? -1 : 0);
+    byte[] bytesBE = new byte[16];
+    for (int i = 0; i < 16 - bytes.length; i++) {
+      bytesBE[i] = padByte;
+    }
+    System.arraycopy(bytes, 0, bytesBE, 16 - bytes.length, bytes.length);
+    return bytesBE;
+  }
+
+  /**
+   * Converts an object, string or binary value to its byte array representation.
+   *
+   * @param value value to parse
+   * @param stats column stats to update
+   * @param columnMetadata column metadata
+   * @return string (byte array) representation
+   */
+  private static String getBinaryValue(
+      Object value, RowBufferStats stats, ColumnMetadata columnMetadata) {
+    AbstractRowBuffer.ColumnLogicalType logicalType =
+        AbstractRowBuffer.ColumnLogicalType.valueOf(columnMetadata.getLogicalType());
+    String str;
+    if (logicalType.isObject()) {
+      switch (logicalType) {
+        case OBJECT:
+          str = DataValidationUtil.validateAndParseObject(value);
+          break;
+        case VARIANT:
+          str = DataValidationUtil.validateAndParseVariant(value);
+          break;
+        case ARRAY:
+          str = DataValidationUtil.validateAndParseArray(value);
+          break;
+        default:
+          throw new SFException(
+              ErrorCode.UNKNOWN_DATA_TYPE, logicalType, columnMetadata.getPhysicalType());
+      }
+    } else if (logicalType == AbstractRowBuffer.ColumnLogicalType.BINARY) {
+      String maxLengthString = columnMetadata.getLength().toString();
+      byte[] bytes =
+          DataValidationUtil.validateAndParseBinary(
+              value, Optional.of(maxLengthString).map(Integer::parseInt));
+      str = new String(bytes, StandardCharsets.UTF_8);
+      stats.addStrValue(str);
+    } else {
+      String maxLengthString = columnMetadata.getLength().toString();
+      str =
+          DataValidationUtil.validateAndParseString(
+              value, Optional.of(maxLengthString).map(Integer::parseInt));
+      stats.addStrValue(str);
+    }
+    return str;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index 739ae5499..44b90987b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -25,7 +25,8 @@
  * Register one or more blobs to the targeted Snowflake table, it will be done using the dedicated
  * thread in order to maintain ordering per channel
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 class RegisterService<T> {
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
index a229f4db3..2aa9d9861 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
@@ -13,7 +13,8 @@
  * Interface for the buffer in the Streaming Ingest channel that holds the un-flushed rows, these
  * rows will be converted to the underlying format implementation for faster processing
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 interface RowBuffer<T> {
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index d2d64db29..7016b23f5 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -30,7 +30,8 @@
 /**
  * The first version of implementation for SnowflakeStreamingIngestChannel
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIngestChannel {
 
@@ -148,9 +149,20 @@ private RowBuffer<T> createRowBuffer(Constants.BdecVersion bdecVersion) {
     // TODO: The circular dependency SnowflakeStreamingIngestChannelInternal <-> RowBuffer
     // (SNOW-657667)
     // can be probably reconsidered
-    //noinspection unchecked
-    return (RowBuffer<T>)
-        new ArrowRowBuffer((SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>) this);
+    switch (bdecVersion) {
+      case ONE:
+      case TWO:
+        //noinspection unchecked
+        return (RowBuffer<T>)
+            new ArrowRowBuffer((SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>) this);
+      case THREE:
+        //noinspection unchecked
+        return (RowBuffer<T>)
+            new ParquetRowBuffer((SnowflakeStreamingIngestChannelInternal<ParquetChunkData>) this);
+      default:
+        throw new SFException(
+            ErrorCode.INTERNAL_ERROR, "Unsupported BDEC format version: " + bdecVersion);
+    }
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 6a3e99382..8be3c4ed3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -82,7 +82,8 @@
  * <li>the channel cache, which contains all the channels that belong to this account
  * <li>the flush service, which schedules and coordinates the flush to Snowflake tables
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStreamingIngestClient {
 
@@ -553,7 +554,7 @@ List<BlobMetadata> getRetryBlobs(
                   .collect(Collectors.toList());
           if (!relevantChunks.isEmpty()) {
             retryBlobs.add(
-                new BlobMetadata(
+                BlobMetadata.createBlobMetadata(
                     blobMetadata.getPath(),
                     blobMetadata.getMD5(),
                     blobMetadata.getVersion(),
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index fed4022e1..68602e0a9 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -83,7 +83,14 @@ public enum BdecVersion {
     ONE(1),
 
     /** Uses Arrow to generate BDEC chunks with {@link ArrowBatchWriteMode#FILE}. */
-    TWO(2);
+    TWO(2),
+
+    /**
+     * Uses Parquet to generate BDEC chunks with {@link
+     * net.snowflake.ingest.streaming.internal.ParquetRowBuffer} (page-level compression). This
+     * version is experimental and WIP at the moment.
+     */
+    THREE(3);
 
     private final byte version;
 
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 5e5e594cd..9f586a04c 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -14,7 +14,6 @@
 import static net.snowflake.ingest.utils.Constants.USER;
 import static net.snowflake.ingest.utils.Constants.WAREHOUSE;
 import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION;
-import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -35,6 +34,7 @@
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.Utils;
 import org.apache.commons.codec.binary.Base64;
 import org.junit.Assert;
@@ -180,7 +180,7 @@ public static KeyPair getKeyPair() throws Exception {
     return keyPair;
   }
 
-  public static Properties getProperties() throws Exception {
+  public static Properties getProperties(Constants.BdecVersion bdecVersion) throws Exception {
     if (profile == null) {
       init();
     }
@@ -195,16 +195,10 @@ public static Properties getProperties() throws Exception {
     props.put(PRIVATE_KEY, privateKeyPem);
     props.put(ROLE, role);
     props.put(ACCOUNT_URL, getAccountURL());
-    props.put(BLOB_FORMAT_VERSION, getBlobFormatVersion());
+    props.put(BLOB_FORMAT_VERSION, bdecVersion.toByte());
     return props;
   }
 
-  private static byte getBlobFormatVersion() {
-    return profile.has(BLOB_FORMAT_VERSION)
-        ? (byte) profile.get(BLOB_FORMAT_VERSION).asInt()
-        : BLOB_FORMAT_VERSION_DEFAULT.toByte();
-  }
-
   /**
    * Create snowflake jdbc connection
    *
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
index c07966c68..5785bd221 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
@@ -57,15 +57,13 @@ public void testFieldNumberAfterFlush() {
   @Test
   public void buildFieldFixedSB1() {
     // FIXED, SB1
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB1");
-    testCol.setNullable(true);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB1")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -80,15 +78,13 @@ public void buildFieldFixedSB1() {
 
   @Test
   public void buildFieldFixedSB2() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB2");
-    testCol.setNullable(false);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB2")
+            .nullable(false)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -103,15 +99,13 @@ public void buildFieldFixedSB2() {
 
   @Test
   public void buildFieldFixedSB4() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB4");
-    testCol.setNullable(true);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB4")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -125,15 +119,13 @@ public void buildFieldFixedSB4() {
 
   @Test
   public void buildFieldFixedSB8() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -147,15 +139,13 @@ public void buildFieldFixedSB8() {
 
   @Test
   public void buildFieldFixedSB16() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB16");
-    testCol.setNullable(true);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     ArrowType expectedType =
@@ -172,15 +162,13 @@ public void buildFieldFixedSB16() {
 
   @Test
   public void buildFieldLobVariant() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("LOB");
-    testCol.setNullable(true);
-    testCol.setLogicalType("VARIANT");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("VARIANT")
+            .physicalType("LOB")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -194,15 +182,13 @@ public void buildFieldLobVariant() {
 
   @Test
   public void buildFieldTimestampNtzSB8() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIMESTAMP_NTZ");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -216,15 +202,13 @@ public void buildFieldTimestampNtzSB8() {
 
   @Test
   public void buildFieldTimestampNtzSB16() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB16");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIMESTAMP_NTZ");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -242,15 +226,13 @@ public void buildFieldTimestampNtzSB16() {
 
   @Test
   public void buildFieldTimestampTzSB8() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIMESTAMP_TZ");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_TZ")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -268,15 +250,13 @@ public void buildFieldTimestampTzSB8() {
 
   @Test
   public void buildFieldTimestampTzSB16() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB16");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIMESTAMP_TZ");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_TZ")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -295,16 +275,14 @@ public void buildFieldTimestampTzSB16() {
   }
 
   @Test
-  public void buildFieldDate() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("DATE");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+  public void buildFieldTimestampDate() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("DATE")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -318,15 +296,13 @@ public void buildFieldDate() {
 
   @Test
   public void buildFieldTimeSB4() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB4");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIME");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB4")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -340,15 +316,13 @@ public void buildFieldTimeSB4() {
 
   @Test
   public void buildFieldTimeSB8() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIME");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -362,15 +336,13 @@ public void buildFieldTimeSB8() {
 
   @Test
   public void buildFieldBoolean() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("BINARY");
-    testCol.setNullable(true);
-    testCol.setLogicalType("BOOLEAN");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("BOOLEAN")
+            .physicalType("BINARY")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -384,15 +356,13 @@ public void buildFieldBoolean() {
 
   @Test
   public void buildFieldRealSB16() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB16");
-    testCol.setNullable(true);
-    testCol.setLogicalType("REAL");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("REAL")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java b/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
new file mode 100644
index 000000000..51d4226bb
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
@@ -0,0 +1,164 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+/** Helper class to build ColumnMetadata */
+public class ColumnMetadataBuilder {
+  private String name;
+  private String type;
+  private String logicalType;
+  private String physicalType;
+  private Integer precision;
+  private Integer scale;
+  private Integer byteLength;
+  private Integer length;
+  private boolean nullable;
+  private String collation;
+
+  /**
+   * Returns a new ColumnMetadata builder
+   *
+   * @return builder
+   */
+  public static ColumnMetadataBuilder newBuilder() {
+    ColumnMetadataBuilder mb = new ColumnMetadataBuilder();
+    mb.name = "testCol";
+    mb.byteLength = 14;
+    mb.length = 11;
+    mb.scale = 0;
+    mb.precision = 4;
+    return mb;
+  }
+
+  /**
+   * Set name
+   *
+   * @param name column name
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder name(String name) {
+    this.name = name;
+    return this;
+  }
+
+  /**
+   * Set type
+   *
+   * @param type type (as defined in ColumnMetadata)
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder type(String type) {
+    this.type = type;
+    return this;
+  }
+
+  /**
+   * Set column logical type
+   *
+   * @param logicalType logical type
+   * @return columnMetadata object
+   */
+  public ColumnMetadataBuilder logicalType(String logicalType) {
+    this.logicalType = logicalType;
+    return this;
+  }
+
+  /**
+   * Set column physical type
+   *
+   * @param physicalType physical type
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder physicalType(String physicalType) {
+    this.physicalType = physicalType;
+    return this;
+  }
+
+  /**
+   * Set column precision
+   *
+   * @param precision precision
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder precision(int precision) {
+    this.precision = precision;
+    return this;
+  }
+
+  /**
+   * Set column scale
+   *
+   * @param scale scale
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder scale(int scale) {
+    this.scale = scale;
+    return this;
+  }
+
+  /**
+   * Set column length in bytes
+   *
+   * @param byteLength length in bytes
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder byteLength(int byteLength) {
+    this.byteLength = byteLength;
+    return this;
+  }
+
+  /**
+   * Set column length (in chars)
+   *
+   * @param length length
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder length(int length) {
+    this.length = length;
+    return this;
+  }
+
+  /**
+   * Set column nullability
+   *
+   * @param nullable nullable
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder nullable(boolean nullable) {
+    this.nullable = nullable;
+    return this;
+  }
+
+  /**
+   * Set column collation
+   *
+   * @param collation collation
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder collation(String collation) {
+    this.collation = collation;
+    return this;
+  }
+
+  /**
+   * Build a columnMetadata object from the set values
+   *
+   * @return columnMetadata object
+   */
+  public ColumnMetadata build() {
+    ColumnMetadata colMetadata = new ColumnMetadata();
+    colMetadata.setName(name);
+    colMetadata.setType(type);
+    colMetadata.setPhysicalType(physicalType);
+    colMetadata.setNullable(nullable);
+    colMetadata.setLogicalType(logicalType);
+    colMetadata.setByteLength(byteLength);
+    colMetadata.setLength(length);
+    colMetadata.setScale(scale);
+    colMetadata.setPrecision(precision);
+    colMetadata.setCollation(collation);
+    return colMetadata;
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 80103e2df..540cd5ea8 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -55,7 +55,8 @@
 public class FlushServiceTest {
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> testContextFactory() {
-    return Arrays.asList(new Object[][] {{ArrowTestContext.createFactory()}});
+    return Arrays.asList(
+        new Object[][] {{ArrowTestContext.createFactory()}, {ParquetTestContext.createFactory()}});
   }
 
   public FlushServiceTest(TestContextFactory<?> testContextFactory) {
@@ -182,11 +183,6 @@ ChannelBuilder setEncryptionKeyId(Long encryptionKeyId) {
         return this;
       }
 
-      ChannelBuilder setOnErrorOption(OpenChannelRequest.OnErrorOption onErrorOption) {
-        this.onErrorOption = onErrorOption;
-        return this;
-      }
-
       SnowflakeStreamingIngestChannelInternal<T> buildAndAdd() {
         SnowflakeStreamingIngestChannelInternal<T> channel =
             createChannel(
@@ -285,6 +281,48 @@ TestContext<VectorSchemaRoot> create() {
     }
   }
 
+  private static class ParquetTestContext extends TestContext<List<List<Object>>> {
+
+    SnowflakeStreamingIngestChannelInternal<List<List<Object>>> createChannel(
+        String name,
+        String dbName,
+        String schemaName,
+        String tableName,
+        String offsetToken,
+        Long channelSequencer,
+        Long rowSequencer,
+        String encryptionKey,
+        Long encryptionKeyId,
+        OpenChannelRequest.OnErrorOption onErrorOption) {
+      return new SnowflakeStreamingIngestChannelInternal<>(
+          name,
+          dbName,
+          schemaName,
+          tableName,
+          offsetToken,
+          channelSequencer,
+          rowSequencer,
+          client,
+          encryptionKey,
+          encryptionKeyId,
+          onErrorOption,
+          Constants.BdecVersion.THREE,
+          null);
+    }
+
+    @Override
+    public void close() {}
+
+    static TestContextFactory<List<List<Object>>> createFactory() {
+      return new TestContextFactory<List<List<Object>>>("Parquet") {
+        @Override
+        TestContext<List<List<Object>>> create() {
+          return new ParquetTestContext();
+        }
+      };
+    }
+  }
+
   TestContextFactory<?> testContextFactory;
   TestContext<?> testContext;
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
new file mode 100644
index 000000000..a477ac977
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
@@ -0,0 +1,480 @@
+package net.snowflake.ingest.streaming.internal;
+
+import java.util.Map;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ParquetTypeGeneratorTest {
+
+  @Test
+  public void buildFieldFixedSB1() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB1")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(4)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
+        .expectedLogicalTypeAnnotation(
+            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB1.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldFixedSB2() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB2")
+            .nullable(false)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(4)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
+        .expectedLogicalTypeAnnotation(
+            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
+        .expectedRepetition(Type.Repetition.REQUIRED)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB2.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldFixedSB4() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB4")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(4)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
+        .expectedLogicalTypeAnnotation(
+            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB4.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldFixedSB8() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(8)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
+        .expectedLogicalTypeAnnotation(
+            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldFixedSB16() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(16)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+        .expectedLogicalTypeAnnotation(
+            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldLobVariant() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("VARIANT")
+            .physicalType("LOB")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(0)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.BINARY)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.stringType())
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.VARIANT.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.LOB.getOrdinal())
+        .assertMatches();
+
+    Assert.assertEquals("1", typeInfo.getMetadata().get(0 + ":obj_enc"));
+  }
+
+  @Test
+  public void buildFieldTimestampNtzSB8() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(8)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldTimestampNtzSB16() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(16)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 38))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldTimestampTzSB8() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_TZ")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(8)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_TZ.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldTimestampTzSB16() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_TZ")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(16)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 38))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_TZ.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldDate() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("DATE")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(0)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.dateType())
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.DATE.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldTimeSB4() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB4")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(4)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 9))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIME.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB4.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldTimeSB8() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(8)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIME.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldBoolean() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("BOOLEAN")
+            .physicalType("BINARY")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(0)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.BOOLEAN)
+        .expectedLogicalTypeAnnotation(null)
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.BOOLEAN.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.BINARY.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldRealSB16() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("REAL")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(0)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.DOUBLE)
+        .expectedLogicalTypeAnnotation(null)
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.REAL.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
+        .assertMatches();
+  }
+
+  /** Builder that helps to assert parquet type info */
+  private static class ParquetTypeInfoAssertionBuilder {
+    private String fieldName;
+    private int fieldId;
+    private PrimitiveType.PrimitiveTypeName primitiveTypeName;
+    private LogicalTypeAnnotation logicalTypeAnnotation;
+    private Type.Repetition repetition;
+    private int typeLength;
+    private String colMetadata;
+    private ParquetTypeGenerator.ParquetTypeInfo typeInfo;
+
+    static ParquetTypeInfoAssertionBuilder newBuilder() {
+      ParquetTypeInfoAssertionBuilder builder = new ParquetTypeInfoAssertionBuilder();
+      return builder;
+    }
+
+    ParquetTypeInfoAssertionBuilder typeInfo(ParquetTypeGenerator.ParquetTypeInfo typeInfo) {
+      this.typeInfo = typeInfo;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedFieldName(String fieldName) {
+      this.fieldName = fieldName;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedFieldId(int fieldId) {
+      this.fieldId = fieldId;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedPrimitiveTypeName(
+        PrimitiveType.PrimitiveTypeName primitiveTypeName) {
+      this.primitiveTypeName = primitiveTypeName;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedLogicalTypeAnnotation(
+        LogicalTypeAnnotation logicalTypeAnnotation) {
+      this.logicalTypeAnnotation = logicalTypeAnnotation;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedRepetition(Type.Repetition repetition) {
+      this.repetition = repetition;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedTypeLength(int typeLength) {
+      this.typeLength = typeLength;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedColMetaData(String colMetaData) {
+      this.colMetadata = colMetaData;
+      return this;
+    }
+
+    void assertMatches() {
+      Type type = typeInfo.getParquetType();
+      Map<String, String> metadata = typeInfo.getMetadata();
+      Assert.assertEquals(fieldName, type.getName());
+      Assert.assertEquals(typeLength, type.asPrimitiveType().getTypeLength());
+      Assert.assertEquals(fieldId, type.asPrimitiveType().getId().intValue());
+
+      Assert.assertEquals(primitiveTypeName, type.asPrimitiveType().getPrimitiveTypeName());
+      Assert.assertEquals(logicalTypeAnnotation, type.getLogicalTypeAnnotation());
+      Assert.assertEquals(repetition, type.getRepetition());
+      Assert.assertEquals(colMetadata, metadata.get(type.getId().toString()));
+    }
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
new file mode 100644
index 000000000..3dc313dc6
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
@@ -0,0 +1,570 @@
+package net.snowflake.ingest.streaming.internal;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.Map;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.parquet.schema.PrimitiveType;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ParquetValueParserTest {
+
+  @Test
+  public void parseValueFixedSB1ToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB1")
+            .scale(0)
+            .precision(2)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            12, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Integer.class)
+        .expectedParsedValue(12)
+        .expectedSize(4.0f)
+        .expectedMinMax(BigInteger.valueOf(12))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueFixedSB2ToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB2")
+            .scale(0)
+            .precision(4)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            1234, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Integer.class)
+        .expectedParsedValue(1234)
+        .expectedSize(4.0f)
+        .expectedMinMax(BigInteger.valueOf(1234))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueFixedSB4ToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB4")
+            .scale(0)
+            .precision(9)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            123456789, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Integer.class)
+        .expectedParsedValue(123456789)
+        .expectedSize(4.0f)
+        .expectedMinMax(BigInteger.valueOf(123456789))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueFixedSB8ToInt64() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB8")
+            .scale(0)
+            .precision(18)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            123456789987654321L, testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Long.class)
+        .expectedParsedValue(123456789987654321L)
+        .expectedSize(8.0f)
+        .expectedMinMax(BigInteger.valueOf(123456789987654321L))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueFixedSB16ToByteArray() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB16")
+            .scale(0)
+            .precision(38)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            new BigDecimal("91234567899876543219876543211234567891"),
+            testCol,
+            PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
+            rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(byte[].class)
+        .expectedParsedValue(
+            ParquetValueParser.getSb16Bytes(
+                new BigInteger("91234567899876543219876543211234567891")))
+        .expectedSize(16.0f)
+        .expectedMinMax(new BigInteger("91234567899876543219876543211234567891"))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueFixedDecimalToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB8")
+            .scale(5)
+            .precision(10)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            new BigDecimal("12345.54321"),
+            testCol,
+            PrimitiveType.PrimitiveTypeName.DOUBLE,
+            rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Double.class)
+        .expectedParsedValue(Double.valueOf("12345.54321"))
+        .expectedSize(8.0f)
+        .expectedMinMax(Double.valueOf("12345.54321"))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueDouble() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("REAL")
+            .physicalType("DOUBLE")
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            12345.54321d, testCol, PrimitiveType.PrimitiveTypeName.DOUBLE, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Double.class)
+        .expectedParsedValue(Double.valueOf(12345.54321))
+        .expectedSize(8.0f)
+        .expectedMinMax(Double.valueOf(12345.54321))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueBoolean() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("BOOLEAN")
+            .physicalType("SB1")
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            true, testCol, PrimitiveType.PrimitiveTypeName.BOOLEAN, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Boolean.class)
+        .expectedParsedValue(true)
+        .expectedSize(1.0f)
+        .expectedMinMax(BigInteger.valueOf(1))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueBinary() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("BINARY")
+            .physicalType("LOB")
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "Length7".getBytes(), testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(String.class)
+        .expectedParsedValue("Length7")
+        .expectedSize(7.0f)
+        .expectedMinMax("Length7")
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueVariantToBinary() {
+    testJsonWithLogicalType("VARIANT");
+  }
+
+  @Test
+  public void parseValueObjectToBinary() {
+    testJsonWithLogicalType("OBJECT");
+  }
+
+  private void testJsonWithLogicalType(String logicalType) {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType(logicalType)
+            .physicalType("BINARY")
+            .nullable(true)
+            .build();
+
+    String var =
+        "{\"key1\":-879869596,\"key2\":\"value2\",\"key3\":null,"
+            + "\"key4\":{\"key41\":0.032437,\"key42\":\"value42\",\"key43\":null}}";
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            var, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(String.class)
+        .expectedParsedValue(var)
+        .expectedSize(var.getBytes().length)
+        .expectedMinMax(null)
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueArrayToBinary() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("ARRAY")
+            .physicalType("BINARY")
+            .nullable(true)
+            .build();
+
+    Map<String, String> input = new HashMap<>();
+    input.put("a", "1");
+    input.put("b", "2");
+    input.put("c", "3");
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            input, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
+
+    String resultArray = "[{\"a\":\"1\",\"b\":\"2\",\"c\":\"3\"}]";
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(String.class)
+        .expectedParsedValue(resultArray)
+        .expectedSize(resultArray.length())
+        .expectedMinMax(null)
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueTimestampNtzSB4Error() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB4")
+            .scale(0) // seconds
+            .precision(9)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    SFException exception =
+        Assert.assertThrows(
+            SFException.class,
+            () ->
+                ParquetValueParser.parseColumnValueToParquet(
+                    "2013-04-28 20:57:00",
+                    testCol,
+                    PrimitiveType.PrimitiveTypeName.INT32,
+                    rowBufferStats));
+    Assert.assertEquals(
+        "Unknown data type for logical: TIMESTAMP_NTZ, physical: SB4.", exception.getMessage());
+  }
+
+  @Test
+  public void parseValueTimestampNtzSB8ToINT64() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB8")
+            .scale(3) // millis
+            .precision(18)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "2013-04-28 20:57:01.000",
+            testCol,
+            PrimitiveType.PrimitiveTypeName.INT64,
+            rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Long.class)
+        .expectedParsedValue(1367182621000L)
+        .expectedSize(8.0f)
+        .expectedMinMax(BigInteger.valueOf(1367182621000L))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueTimestampNtzSB16ToByteArray() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB16")
+            .nullable(true)
+            .scale(9) // nanos
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "2022-09-18 22:05:07.123456789",
+            testCol,
+            PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
+            rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(byte[].class)
+        .expectedParsedValue(
+            ParquetValueParser.getSb16Bytes(BigInteger.valueOf(1663538707123456789L)))
+        .expectedSize(16.0f)
+        .expectedMinMax(BigInteger.valueOf(1663538707123456789L))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueDateToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("DATE")
+            .physicalType("SB4")
+            .scale(0) // seconds
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "2021-01-01", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Integer.class)
+        .expectedParsedValue(Integer.valueOf(18628))
+        .expectedSize(4.0f)
+        .expectedMinMax(BigInteger.valueOf(18628))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueTimeSB4ToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB4")
+            .scale(0) // seconds
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "01:00:00", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Integer.class)
+        .expectedParsedValue(3600)
+        .expectedSize(4.0f)
+        .expectedMinMax(BigInteger.valueOf(3600))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueTimeSB8ToInt64() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB8")
+            .scale(3) // milliseconds
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "01:00:00.123", testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Long.class)
+        .expectedParsedValue(3600123L)
+        .expectedSize(8.0f)
+        .expectedMinMax(BigInteger.valueOf(3600123))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueTimeSB16Error() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB16")
+            .scale(9) // nanos
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    SFException exception =
+        Assert.assertThrows(
+            SFException.class,
+            () ->
+                ParquetValueParser.parseColumnValueToParquet(
+                    "11:00:00.12345678",
+                    testCol,
+                    PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
+                    rowBufferStats));
+    Assert.assertEquals(
+        "Unknown data type for logical: TIME, physical: SB16.", exception.getMessage());
+  }
+
+  /** Builder that helps to assert parsing of values to parquet types */
+  private static class ParquetValueParserAssertionBuilder {
+    private ParquetValueParser.ParquetBufferValue parquetBufferValue;
+    private RowBufferStats rowBufferStats;
+    private Class valueClass;
+    private Object value;
+    private float size;
+    private Object minMaxStat;
+
+    static ParquetValueParserAssertionBuilder newBuilder() {
+      ParquetValueParserAssertionBuilder builder = new ParquetValueParserAssertionBuilder();
+      return builder;
+    }
+
+    ParquetValueParserAssertionBuilder parquetBufferValue(
+        ParquetValueParser.ParquetBufferValue parquetBufferValue) {
+      this.parquetBufferValue = parquetBufferValue;
+      return this;
+    }
+
+    ParquetValueParserAssertionBuilder rowBufferStats(RowBufferStats rowBufferStats) {
+      this.rowBufferStats = rowBufferStats;
+      return this;
+    }
+
+    ParquetValueParserAssertionBuilder expectedValueClass(Class valueClass) {
+      this.valueClass = valueClass;
+      return this;
+    }
+
+    ParquetValueParserAssertionBuilder expectedParsedValue(Object value) {
+      this.value = value;
+      return this;
+    }
+
+    ParquetValueParserAssertionBuilder expectedSize(float size) {
+      this.size = size;
+      return this;
+    }
+
+    public ParquetValueParserAssertionBuilder expectedMinMax(Object minMaxStat) {
+      this.minMaxStat = minMaxStat;
+      return this;
+    }
+
+    void assertMatches() {
+      Assert.assertEquals(valueClass, parquetBufferValue.getValue().getClass());
+      System.out.println("parquetBufferValue = " + parquetBufferValue.getValue().toString());
+      if (valueClass.equals(byte[].class)) {
+        Assert.assertArrayEquals((byte[]) value, (byte[]) parquetBufferValue.getValue());
+      } else {
+        Assert.assertEquals(value, parquetBufferValue.getValue());
+      }
+      Assert.assertEquals(size, parquetBufferValue.getSize(), 0);
+      if (minMaxStat instanceof BigInteger) {
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinIntValue());
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxIntValue());
+        return;
+      } else if (minMaxStat instanceof String || valueClass.equals(String.class)) {
+        // String can have null min/max stats for variant data types
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinColStrValue());
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxColStrValue());
+        return;
+      } else if (minMaxStat instanceof Double || minMaxStat instanceof BigDecimal) {
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinRealValue());
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxRealValue());
+        return;
+      }
+      throw new IllegalArgumentException("Unknown data type for min stat");
+    }
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 7bd986e8a..8d28c0cb9 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -26,7 +26,11 @@
 public class RowBufferTest {
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> bdecVersion() {
-    return Arrays.asList(new Object[][] {{"Arrow", Constants.BdecVersion.ONE}});
+    return Arrays.asList(
+        new Object[][] {
+          {"Arrow", Constants.BdecVersion.ONE},
+          {"Parquet", Constants.BdecVersion.THREE}
+        });
   }
 
   private final Constants.BdecVersion bdecVersion;
@@ -1029,6 +1033,7 @@ private void testE2EBinaryHelper(OpenChannelRequest.OnErrorOption onErrorOption)
     colBinary.setPhysicalType("LOB");
     colBinary.setNullable(true);
     colBinary.setLogicalType("BINARY");
+    colBinary.setLength(32);
     colBinary.setScale(0);
 
     innerBuffer.setupSchema(Collections.singletonList(colBinary));
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 175db5fb8..c5189bcde 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -9,14 +9,15 @@
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
-import java.util.Random;
 import java.util.TimeZone;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
@@ -31,6 +32,7 @@
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.ParameterProvider;
 import net.snowflake.ingest.utils.SFException;
@@ -38,9 +40,23 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 /** Example streaming ingest sdk integration test */
+@RunWith(Parameterized.class)
 public class StreamingIngestIT {
+  @Parameterized.Parameters(name = "{0}")
+  public static Collection<Object[]> bdecVersion() {
+    return Arrays.asList(
+        new Object[][] {
+          {"Arrow", Constants.BdecVersion.ONE},
+          // TODO: uncomment once SNOW-659721 is deployed and we set the parameter
+          // DISABLE_PARQUET_CACHE to true for the test account
+          // {"Parquet", Constants.BdecVersion.THREE}
+        });
+  }
+
   private static final String TEST_TABLE = "STREAMING_INGEST_TEST_TABLE";
   private static final String TEST_DB_PREFIX = "STREAMING_INGEST_TEST_DB";
   private static final String TEST_SCHEMA = "STREAMING_INGEST_TEST_SCHEMA";
@@ -55,11 +71,23 @@ public class StreamingIngestIT {
   private Connection jdbcConnection;
   private String testDb;
 
+  private final Constants.BdecVersion bdecVersion;
+
+  public StreamingIngestIT(
+      @SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
+    this.bdecVersion = bdecVersion;
+  }
+
   @Before
   public void beforeAll() throws Exception {
     testDb = TEST_DB_PREFIX + "_" + UUID.randomUUID().toString().substring(0, 4);
     // Create a streaming ingest client
     jdbcConnection = TestUtils.getConnection(true);
+    if (bdecVersion == Constants.BdecVersion.THREE) {
+      // TODO: encryption and interleaved mode are not yet supported by server side's Parquet
+      // scanner if local file cache is enabled (SNOW-656500)
+      jdbcConnection.createStatement().execute("alter session set disable_parquet_cache=true;");
+    }
     jdbcConnection
         .createStatement()
         .execute(String.format("create or replace database %s;", testDb));
@@ -78,7 +106,7 @@ public void beforeAll() throws Exception {
         .createStatement()
         .execute(String.format("use warehouse %s", TestUtils.getWarehouse()));
 
-    prop = TestUtils.getProperties();
+    prop = TestUtils.getProperties(bdecVersion);
     if (prop.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       prop.setProperty(ROLE, "ACCOUNTADMIN");
     }
@@ -1042,7 +1070,7 @@ private void verifyTableRowCount(int rowNumber, String tableName) {
   }
 
   @Test
-  public void testDataTypes() {
+  public void testDataTypes() throws SQLException, ParseException {
     String tableName = "t_data_types";
     try {
       jdbcConnection
@@ -1080,94 +1108,195 @@ public void testDataTypes() {
 
     SnowflakeStreamingIngestChannel channel = openChannel(tableName, "CHANNEL");
 
-    Random r = new Random();
-    for (int val = 0; val < 10000; val++) {
-      verifyInsertValidationResponse(channel.insertRow(getRandomRow(r), Integer.toString(val)));
-    }
-    waitChannelFlushed(channel, 10000);
+    Map<String, Object> negRow = getNegRow();
+    verifyInsertValidationResponse(channel.insertRow(negRow, Integer.toString(0)));
+
+    Map<String, Object> nullRow = getNullRow();
+    verifyInsertValidationResponse(channel.insertRow(nullRow, Integer.toString(1)));
+
+    Map<String, Object> posRow = getPosRow();
+    verifyInsertValidationResponse(channel.insertRow(posRow, Integer.toString(2)));
+
+    waitChannelFlushed(channel, 3);
+    verifyTableRowCount(3, tableName);
+
+    ResultSet result =
+        jdbcConnection
+            .createStatement()
+            .executeQuery(
+                String.format(
+                    "select * from %s.%s.%s order by num", testDb, TEST_SCHEMA, tableName));
+
+    result.next(); // negRow
+    assertNegRow(negRow, result);
+
+    result.next(); // nullRow
+    assertNullRow(nullRow, result);
+
+    result.next(); // posRow
+    assertPosRow(posRow, result);
   }
 
-  private static Map<String, Object> getRandomRow(Random r) {
-    Map<String, Object> row = new HashMap<>();
-    row.put("num", r.nextInt());
-    row.put("num_10_5", nextFloat(r));
-    row.put(
-        "num_38_0",
-        new BigDecimal("" + nextLongOfPrecision(r, 18) + Math.abs(nextLongOfPrecision(r, 18))));
-    row.put("num_2_0", r.nextInt(100));
-    row.put("num_4_0", r.nextInt(10000));
-    row.put("num_9_0", r.nextInt(1000000000));
-    row.put("num_18_0", nextLongOfPrecision(r, 18));
-    row.put("num_float", nextFloat(r));
-    row.put("str_varchar", nextString(r));
-    row.put("bin", nextBytes(r));
-    row.put("bl", r.nextBoolean());
-    row.put("var", nextJson(r));
-    row.put("obj", nextJson(r));
-    row.put("arr", Arrays.asList(r.nextInt(100), r.nextInt(100), r.nextInt(100)));
-    row.put(
-        "epochdays",
-        String.valueOf(Math.abs(r.nextInt()) % (18963 * 24 * 60 * 60))); // DATE, 02.12.2021
-    row.put(
-        "timesec",
-        String.valueOf(
-            (r.nextInt(11) * 60 * 60 + r.nextInt(59) * 60 + r.nextInt(59)) * 10000
-                + r.nextInt(9999))); // TIME(4), 05:12:43.4536
-    row.put(
-        "timenano",
-        String.valueOf(
-            (14 * 60 * 60 + 26 * 60 + 34) * 1000000000L
-                + 437582643)); // TIME(9), 14:26:34.437582643
-    row.put(
-        "epochsec",
-        String.valueOf(
-            Math.abs(r.nextLong()) % 1638459438)); // TIMESTAMP_LTZ(0), 02.12.2021 15:37:18
-    row.put(
-        "epochnano",
-        String.format(
-            "%d%d",
-            Math.abs(r.nextInt()) % 1999999999,
-            100000000
-                + Math.abs(
-                    r.nextInt(899999999)))); // TIMESTAMP_LTZ(9), 18.05.2033 03:33:19.999999999
-    return row;
+  private Map<String, Object> getNegRow() {
+    Map<String, Object> negRow = new HashMap<>();
+    negRow.put("num", -123);
+    negRow.put("num_10_5", new BigDecimal("-12345.54321"));
+    negRow.put("num_38_0", new BigDecimal("-91234567899876543219876543211234567891"));
+    negRow.put("num_2_0", -12);
+    negRow.put("num_4_0", -1234);
+    negRow.put("num_9_0", -123456789);
+    negRow.put("num_18_0", -123456789987654321L);
+    negRow.put("num_float", -12345.54321f);
+    negRow.put("str_varchar", "zxccvbnm,.");
+    negRow.put("bin", new byte[] {1, 2, 3});
+    negRow.put("bl", true);
+    negRow.put("var", "{}");
+    negRow.put("obj", "{}");
+    negRow.put("arr", Arrays.asList());
+    negRow.put("epochdays", "0");
+    negRow.put("epochsec", "0");
+    negRow.put("epochnano", "0");
+    negRow.put("timesec", "0");
+    negRow.put("timenano", "0");
+
+    return negRow;
   }
 
-  private static long nextLongOfPrecision(Random r, int precision) {
-    return r.nextLong() % Math.round(Math.pow(10, precision));
+  private Map<String, Object> getNullRow() {
+    Map<String, Object> nullRow = new HashMap<>();
+    nullRow.put("num", 0);
+    nullRow.put("num_10_5", new BigDecimal("0.00000"));
+    nullRow.put("num_38_0", new BigDecimal("0"));
+    nullRow.put("num_2_0", 0);
+    nullRow.put("num_4_0", 0);
+    nullRow.put("num_9_0", 0);
+    nullRow.put("num_18_0", 0L);
+    nullRow.put("num_float", 0.0f);
+    nullRow.put("str_varchar", null);
+    nullRow.put("bin", null);
+    nullRow.put("bl", false);
+    nullRow.put("var", null);
+    nullRow.put("obj", null);
+    nullRow.put("arr", null);
+    nullRow.put("epochdays", null);
+    nullRow.put("epochsec", null);
+    nullRow.put("epochnano", null);
+    nullRow.put("timesec", null);
+    nullRow.put("timenano", null);
+
+    return nullRow;
   }
 
-  private static String nextString(Random r) {
-    return new String(nextBytes(r));
+  private Map<String, Object> getPosRow() {
+    Map<String, Object> posRow = new HashMap<>();
+    posRow.put("num", 123);
+    posRow.put("num_10_5", new BigDecimal("12345.54321"));
+    posRow.put("num_38_0", new BigDecimal("91234567899876543219876543211234567891"));
+    posRow.put("num_2_0", 12);
+    posRow.put("num_4_0", 1234);
+    posRow.put("num_9_0", 123456789);
+    posRow.put("num_18_0", 123456789987654321L);
+    posRow.put("num_float", 12345.54321f);
+    posRow.put("str_varchar", "abcd123456.");
+    posRow.put("bin", new byte[] {4, 5, 6});
+    posRow.put("bl", true);
+    posRow.put(
+        "var",
+        "{ \"a\": 1, \"b\": \"qwerty\", \"c\": null, \"d\": { \"e\": 2, \"f\": \"asdf\", \"g\":"
+            + " null } }");
+    posRow.put(
+        "obj",
+        "{ \"a\": 1, \"b\": \"qwerty\", \"c\": null, \"d\": { \"e\": 2, \"f\": \"asdf\", \"g\":"
+            + " null } }");
+    posRow.put("arr", Arrays.asList("{ \"a\": 1}", "{ \"b\": 2 }", "{ \"c\": 3 }"));
+    posRow.put("epochdays", "2022-09-18 20:05:07"); // DATE, 18.09.2022
+    posRow.put("epochsec", "2022-09-18 20:05:07"); // TIMESTAMP_NTZ(0)
+    posRow.put("epochnano", "2022-09-18 20:05:07.999999999"); // TIMESTAMP_NTZ(9)
+    posRow.put("timesec", "01:00:01.999999999"); // TIME(0)
+    posRow.put("timenano", "01:00:01.999999999"); // TIME(9)
+
+    return posRow;
   }
 
-  private static byte[] nextBytes(Random r) {
-    byte[] bin = new byte[128];
-    r.nextBytes(bin);
-    for (int i = 0; i < bin.length; i++) {
-      bin[i] = (byte) (Math.abs(bin[i]) % 25 + 97); // ascii letters
-    }
-    return bin;
+  private void assertNegRow(Map<String, Object> expectedNegRow, ResultSet actualResult)
+      throws SQLException {
+    assertNonTimeAndVarFields(expectedNegRow, actualResult);
+    Assert.assertEquals("{}", actualResult.getString("VAR"));
+    Assert.assertEquals("{}", actualResult.getString("OBJ"));
+    Assert.assertEquals("[]", actualResult.getString("ARR"));
+    Assert.assertEquals(0, actualResult.getDate("EPOCHDAYS").getTime());
+    Assert.assertEquals(0, actualResult.getTimestamp("EPOCHSEC").getTime());
+    Assert.assertEquals(0, actualResult.getTimestamp("EPOCHNANO").getNanos());
+    Assert.assertEquals(0, actualResult.getTimestamp("EPOCHNANO").getTime());
+    Assert.assertEquals(0, actualResult.getTimestamp("TIMESEC").getTime());
+    Assert.assertEquals(0, actualResult.getTimestamp("TIMENANO").getNanos());
+    Assert.assertEquals(0, actualResult.getTimestamp("TIMENANO").getTime());
+  }
+
+  private void assertNullRow(Map<String, Object> expectedNullRow, ResultSet actualResult)
+      throws SQLException {
+    Assert.assertNotNull(actualResult);
+    assertNonTimeAndVarFields(expectedNullRow, actualResult);
+    Assert.assertEquals(null, actualResult.getString("VAR"));
+    Assert.assertEquals(null, actualResult.getString("OBJ"));
+    Assert.assertEquals(null, actualResult.getString("ARR"));
+    Assert.assertEquals(null, actualResult.getDate("EPOCHDAYS"));
+    Assert.assertEquals(null, actualResult.getTimestamp("EPOCHSEC"));
+    Assert.assertEquals(null, actualResult.getTimestamp("EPOCHNANO"));
+    Assert.assertEquals(null, actualResult.getTimestamp("TIMESEC"));
+    Assert.assertEquals(null, actualResult.getTimestamp("TIMENANO"));
   }
 
-  private static double nextFloat(Random r) {
-    return (r.nextLong() % Math.round(Math.pow(10, 10))) / 100000d;
+  private void assertPosRow(Map<String, Object> expectedPosRow, ResultSet actualResult)
+      throws SQLException {
+    String formattedJSON =
+        "{\n"
+            + "  \"a\": 1,\n"
+            + "  \"b\": \"qwerty\",\n"
+            + "  \"c\": null,\n"
+            + "  \"d\": {\n"
+            + "    \"e\": 2,\n"
+            + "    \"f\": \"asdf\",\n"
+            + "    \"g\": null\n"
+            + "  }\n"
+            + "}";
+
+    String formattedArray =
+        "[\n"
+            + "  \"{ \\\"a\\\": 1}\",\n"
+            + "  \"{ \\\"b\\\": 2 }\",\n"
+            + "  \"{ \\\"c\\\": 3 }\"\n"
+            + "]";
+    Assert.assertNotNull(actualResult);
+    assertNonTimeAndVarFields(expectedPosRow, actualResult);
+    Assert.assertEquals(formattedJSON, actualResult.getString("VAR"));
+    Assert.assertEquals(formattedJSON, actualResult.getString("OBJ"));
+    Assert.assertEquals(formattedArray, actualResult.getString("ARR"));
+    Assert.assertEquals(
+        1663459200000l, actualResult.getDate("EPOCHDAYS").getTime()); // in ms, 18.09.2022 00:00:00
+    Assert.assertEquals(1663531507000L, actualResult.getTimestamp("EPOCHSEC").getTime());
+    Assert.assertEquals(999999999L, actualResult.getTimestamp("EPOCHNANO").getNanos());
+    // 1663531507000 ms + 999 ms (from ns)
+    Assert.assertEquals(1663531507999L, actualResult.getTimestamp("EPOCHNANO").getTime());
+    Assert.assertEquals(3601000, actualResult.getTimestamp("TIMESEC").getTime()); // 1h + 1s in ms
+    Assert.assertEquals(999999999, actualResult.getTimestamp("TIMENANO").getNanos());
+    // 1h + 1s + 999 ms (from ns)
+    Assert.assertEquals(3601999, actualResult.getTimestamp("TIMENANO").getTime());
   }
 
-  private static String nextJson(Random r) {
-    return String.format(
-        "{ \"%s\": %d, \"%s\": \"%s\", \"%s\": null, \"%s\": { \"%s\": %f, \"%s\": \"%s\", \"%s\":"
-            + " null } }",
-        nextString(r),
-        r.nextInt(),
-        nextString(r),
-        nextString(r),
-        nextString(r),
-        nextString(r),
-        nextString(r),
-        r.nextFloat(),
-        nextString(r),
-        nextString(r),
-        nextString(r));
+  private void assertNonTimeAndVarFields(Map<String, Object> row, ResultSet result)
+      throws SQLException {
+    Assert.assertNotNull(result);
+    Assert.assertEquals(row.get("num"), result.getInt("NUM"));
+    Assert.assertEquals(row.get("num_10_5"), result.getBigDecimal("NUM_10_5"));
+    Assert.assertEquals(row.get("num_38_0"), result.getBigDecimal("NUM_38_0"));
+    Assert.assertEquals(row.get("num_2_0"), result.getInt("NUM_2_0"));
+    Assert.assertEquals(row.get("num_4_0"), result.getInt("NUM_4_0"));
+    Assert.assertEquals(row.get("num_9_0"), result.getInt("NUM_9_0"));
+    Assert.assertEquals((long) row.get("num_18_0"), result.getLong("NUM_18_0"));
+    Assert.assertEquals((float) row.get("num_float"), result.getFloat("NUM_FLOAT"), 0);
+    Assert.assertEquals(row.get("str_varchar"), result.getString("STR_VARCHAR"));
+    Assert.assertArrayEquals((byte[]) row.get("bin"), result.getBytes("BIN"));
+    Assert.assertEquals(row.get("bl"), result.getBoolean("BL"));
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index 0ea0e5de4..31afda1f0 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -9,6 +9,8 @@
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
@@ -19,12 +21,27 @@
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.SFException;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
+@RunWith(Parameterized.class)
 public abstract class AbstractDataTypeTest {
+  @Parameterized.Parameters(name = "{0}")
+  public static Collection<Object[]> bdecVersion() {
+    return Arrays.asList(
+        new Object[][] {
+          {"Arrow", Constants.BdecVersion.ONE},
+          // TODO: uncomment once SNOW-659721 is deployed and we set the parameter
+          // DISABLE_PARQUET_CACHE to true for the test account
+          // {"Parquet", Constants.BdecVersion.THREE}
+        });
+  }
+
   private static final String SOURCE_COLUMN_NAME = "source";
   private static final String VALUE_COLUMN_NAME = "value";
 
@@ -52,6 +69,13 @@ protected String randomString() {
 
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
+  private final Constants.BdecVersion bdecVersion;
+
+  public AbstractDataTypeTest(
+      @SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
+    this.bdecVersion = bdecVersion;
+  }
+
   @Before
   public void before() throws Exception {
     databaseName = String.format("SDK_DATATYPE_COMPATIBILITY_IT_%s", randomString());
@@ -65,7 +89,13 @@ public void before() throws Exception {
 
     conn.createStatement().execute(String.format("use warehouse %s;", TestUtils.getWarehouse()));
 
-    Properties props = TestUtils.getProperties();
+    if (bdecVersion == Constants.BdecVersion.THREE) {
+      // TODO: encryption and interleaved mode are not yet supported by server side's Parquet
+      // scanner if local file cache is enabled (SNOW-656500)
+      conn.createStatement().execute("alter session set disable_parquet_cache=true;");
+    }
+
+    Properties props = TestUtils.getProperties(bdecVersion);
     if (props.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       props.setProperty(ROLE, "ACCOUNTADMIN");
     }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
index f0b0d8199..a7b9d6cd7 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
@@ -1,10 +1,15 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Ignore;
 import org.junit.Test;
 
 public class BinaryIT extends AbstractDataTypeTest {
 
+  public BinaryIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testBinarySimple() throws Exception {
     testJdbcTypeCompatibility("BINARY", new byte[0], new ByteArrayProvider());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
index bad5eac16..3a9445ea2 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
@@ -8,6 +8,7 @@
 import java.time.OffsetTime;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Ignore;
 import org.junit.Test;
 
@@ -17,6 +18,10 @@
  */
 public class DateTimeIT extends AbstractDataTypeTest {
 
+  public DateTimeIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testTimestampWithTimeZone() throws Exception {
     useLosAngelesTimeZone();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
index 1fdd2ad5e..48936b950 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
@@ -2,9 +2,15 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class LogicalTypesIT extends AbstractDataTypeTest {
+
+  public LogicalTypesIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testLogicalTypes() throws Exception {
     // Test booleans
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
index 57867427d..22c59d3ea 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
@@ -1,9 +1,15 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
 import java.util.Arrays;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class NullIT extends AbstractDataTypeTest {
+
+  public NullIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testNullIngestion() throws Exception {
     for (String type :
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
index 1b1c8d280..64c9bc3fc 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
@@ -2,9 +2,15 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class NumericTypesIT extends AbstractDataTypeTest {
+
+  public NumericTypesIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testIntegers() throws Exception {
     // test bytes
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
index 3c4ddde22..1195d2bfc 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
@@ -15,6 +15,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -24,6 +25,10 @@ public class SemiStructuredIT extends AbstractDataTypeTest {
   // server-side representation. Validation leaves a small buffer for this difference.
   private static final int MAX_ALLOWED_LENGTH = 16 * 1024 * 1024 - 64;
 
+  public SemiStructuredIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testVariant() throws Exception {
     // Test dates
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
index 4bcf39519..198d1c99c 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
@@ -2,10 +2,16 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Ignore;
 import org.junit.Test;
 
 public class StringsIT extends AbstractDataTypeTest {
+
+  public StringsIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testStrings() throws Exception {
     testJdbcTypeCompatibility("VARCHAR", "", new StringProvider());

From 8019fb96f6a5498191fa7eff3fcd1eb432aab6a3 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 20 Oct 2022 12:57:02 -0700
Subject: [PATCH 081/356] Revert "Introduce parquet file generator  (#251)"
 (#269)

This reverts commit 0596f5a, due to a few vulnerabilities from Snyk and Dependapot
---
 pom.xml                                       |  22 -
 scripts/check_content.sh                      |  21 +-
 .../streaming/internal/ArrowRowBuffer.java    |  10 +-
 .../streaming/internal/BlobMetadata.java      |  17 +-
 .../internal/BlobMetadataWithBdecVersion.java |  25 -
 .../streaming/internal/ChannelCache.java      |   3 +-
 .../internal/DataValidationUtil.java          |  11 -
 .../streaming/internal/FlushService.java      |   7 +-
 .../ingest/streaming/internal/Flusher.java    |   3 +-
 .../streaming/internal/ParquetChunkData.java  |  25 -
 .../streaming/internal/ParquetFlusher.java    | 348 -----------
 .../streaming/internal/ParquetRowBuffer.java  | 222 -------
 .../internal/ParquetTypeGenerator.java        | 292 ---------
 .../internal/ParquetValueParser.java          | 323 ----------
 .../streaming/internal/RegisterService.java   |   3 +-
 .../ingest/streaming/internal/RowBuffer.java  |   3 +-
 ...owflakeStreamingIngestChannelInternal.java |  20 +-
 ...nowflakeStreamingIngestClientInternal.java |   5 +-
 .../net/snowflake/ingest/utils/Constants.java |   9 +-
 .../java/net/snowflake/ingest/TestUtils.java  |  12 +-
 .../streaming/internal/ArrowBufferTest.java   | 242 ++++----
 .../internal/ColumnMetadataBuilder.java       | 164 -----
 .../streaming/internal/FlushServiceTest.java  |  50 +-
 .../internal/ParquetTypeGeneratorTest.java    | 480 ---------------
 .../internal/ParquetValueParserTest.java      | 570 ------------------
 .../streaming/internal/RowBufferTest.java     |   7 +-
 .../streaming/internal/StreamingIngestIT.java | 289 +++------
 .../datatypes/AbstractDataTypeTest.java       |  32 +-
 .../internal/datatypes/BinaryIT.java          |   5 -
 .../internal/datatypes/DateTimeIT.java        |   5 -
 .../internal/datatypes/LogicalTypesIT.java    |   6 -
 .../streaming/internal/datatypes/NullIT.java  |   6 -
 .../internal/datatypes/NumericTypesIT.java    |   6 -
 .../internal/datatypes/SemiStructuredIT.java  |   5 -
 .../internal/datatypes/StringsIT.java         |   6 -
 35 files changed, 262 insertions(+), 2992 deletions(-)
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
 delete mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
 delete mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
 delete mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java

diff --git a/pom.xml b/pom.xml
index 23726b1d4..cb6b2eda0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -35,13 +35,6 @@
       <url>https://github.com/snowflakedb/snowflake-ingest-java/tree/master</url>
     </scm>
 
-    <repositories>
-        <repository>
-            <id>cloudera-repo</id>
-            <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
-        </repository>
-    </repositories>
-
     <!-- Set our Language Level to Java 8 -->
     <properties>
         <maven.compiler.source>1.8</maven.compiler.source>
@@ -50,7 +43,6 @@
         <jacoco.skip.instrument>true</jacoco.skip.instrument>
         <jacoco.version>0.8.5</jacoco.version>
         <arrow.version>9.0.0</arrow.version>
-        <hadoop.version>2.10.1</hadoop.version>
     </properties>
 
     <build>
@@ -300,20 +292,6 @@
             <version>${arrow.version}</version>
         </dependency>
 
-        <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop -->
-        <dependency>
-            <groupId>org.apache.parquet</groupId>
-            <artifactId>parquet-hadoop</artifactId>
-            <version>1.10.99.7.2.15.0-147</version>
-        </dependency>
-
-        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-common</artifactId>
-            <version>3.1.1.7.2.15.0-147</version>
-        </dependency>
-
         <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-core -->
         <dependency>
             <groupId>io.dropwizard.metrics</groupId>
diff --git a/scripts/check_content.sh b/scripts/check_content.sh
index e4cfe7f2c..9e5b1c404 100755
--- a/scripts/check_content.sh
+++ b/scripts/check_content.sh
@@ -5,26 +5,7 @@
 set -o pipefail
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-if jar tvf $DIR/../target/snowflake-ingest-sdk.jar  | awk '{print $8}' | grep -v -E "^(net|com)/snowflake" \
-    | grep -v -E "(com|net)/\$" | grep -v -E "^META-INF" | grep -v -E "^mozilla" | grep -v mime.types \
-    | grep -v project.properties | grep -v -E "javax" | grep -v -E "^org/" | grep -v -E "^com/google" \
-    | grep -v -E "^com/sun" | grep -v "log4j.properties" | grep -v "git.properties" | grep -v "io/" \
-    | grep -v "codegen/" | grep -v "com/codahale/" | grep -v "com/ibm/" | grep -v "LICENSE" | grep -v "aix/" \
-    | grep -v "darwin/" | grep -v "win/" | grep -v "freebsd/" | grep -v "linux/" | grep -v "com/github/" \
-    | grep -v -E "shaded/" | grep -v "webapps/"  | grep -v "microsoft/" | grep -v "com/ctc/" | grep -v "jersey/" \
-    | grep -v "edu/" | grep -v "com/jcraft/" | grep -v "contribs/" | grep -v "com/zaxxer/"  | grep -v "com/squareup/" \
-    | grep -v "com/thoughtworks/" | grep -v "com/jamesmurty/"  | grep -v "net/iharder/" \
-    | grep -v -E "^core-default.xml" \
-    | grep -v "yarn-version-info.properties" | grep -v "yarn-version-info.properties" \
-    | grep -v "about.html" | grep -v "jetty-dir.css" \
-    | grep -v "krb5_udp-template.conf" | grep -v "krb5-template.conf" \
-    | grep -v "ccache.txt" | grep -v "keytab.txt" \
-    | grep -v "common-version-info.properties" | grep -v "LocalizedFormats_fr.properties" \
-    | grep -v "org.apache.hadoop.application-classloader.properties" | grep -v "assets/" | grep -v "ehcache-core.xsd" \
-    | grep -v "ehcache-107ext.xsd" | grep -v "parquet.thrift" | grep -v "mapred-default.xml" \
-    | grep -v "yarn-default.xml" | grep -v ".keep"  | grep -v "NOTICE" | grep -v "digesterRules.xml" \
-    | grep -v "properties.dtd" | grep -v "PropertyList-1.0.dtd"  \
-    ; then
+if jar tvf $DIR/../target/snowflake-ingest-sdk.jar  | awk '{print $8}' | grep -v -E "^(net|com)/snowflake" | grep -v -E "(com|net)/\$" | grep -v -E "^META-INF" | grep -v -E "^mozilla" | grep -v mime.types | grep -v project.properties | grep -v -E "javax" | grep -v -E "^org/" | grep -v -E "^com/google" | grep -v -E "^com/sun" | grep -v "log4j.properties" | grep -v "git.properties" | grep -v "io/" | grep -v "codegen/" | grep -v "com/codahale/" | grep -v "com/ibm/" | grep -v "LICENSE" | grep -v "aix/" | grep -v "darwin/" | grep -v "win/" | grep -v "freebsd/" | grep -v "linux/" | grep -v "com/github/"; then
   echo "[ERROR] Ingest SDK jar includes class not under the snowflake namespace"
   exit 1
 fi
\ No newline at end of file
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 2d7d17a92..38b4583a9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -471,7 +471,15 @@ private float convertRowToArrow(
             // vector.setSafe requires the BigDecimal input scale explicitly match its scale
             inputAsBigDecimal = inputAsBigDecimal.setScale(columnScale, RoundingMode.HALF_UP);
 
-            DataValidationUtil.checkValueInRange(inputAsBigDecimal, columnScale, columnPrecision);
+            if (inputAsBigDecimal.abs().compareTo(BigDecimal.TEN.pow(columnPrecision - columnScale))
+                >= 0) {
+              throw new SFException(
+                  ErrorCode.INVALID_ROW,
+                  inputAsBigDecimal,
+                  String.format(
+                      "Number out of representable exclusive range of (-1e%s..1e%s)",
+                      columnPrecision - columnScale, columnPrecision - columnScale));
+            }
 
             if (columnScale != 0 || physicalType == ColumnPhysicalType.SB16) {
               ((DecimalVector) vector).setSafe(curRowIndex, inputAsBigDecimal);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index f90ba5055..2818a392a 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -49,16 +49,9 @@ List<ChunkMetadata> getChunks() {
     return this.chunks;
   }
 
-  /**
-   * Create {@link BlobMetadata} in case of {@link Constants.BdecVersion#ONE} and {@link
-   * BlobMetadataWithBdecVersion} otherwise to send BDEC version to server side.
-   */
-  static BlobMetadata createBlobMetadata(
-      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks) {
-    // TODO SNOW-659721: Unify BlobMetadata with BlobMetadataWithBdecVersion once server side
-    // BdecVersion in production
-    return bdecVersion == Constants.BdecVersion.ONE
-        ? new BlobMetadata(path, md5, bdecVersion, chunks)
-        : new BlobMetadataWithBdecVersion(path, md5, bdecVersion, chunks);
-  }
+  // TODO: send the bdec_version once server side supports this in production
+  //  @JsonProperty("bdec_version")
+  //  byte getVersionByte() {
+  //    return bdecVersion.toByte();
+  //  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
deleted file mode 100644
index 7f9c602e3..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import java.util.List;
-import net.snowflake.ingest.utils.Constants;
-
-/**
- * Same as {@link BlobMetadata} but with {@link Constants.BdecVersion} to send it to server side if
- * the version is greater than {@link Constants.BdecVersion#ONE}.
- */
-class BlobMetadataWithBdecVersion extends BlobMetadata {
-  BlobMetadataWithBdecVersion(
-      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks) {
-    super(path, md5, bdecVersion, chunks);
-  }
-
-  @JsonProperty("bdec_version")
-  byte getVersionByte() {
-    return getVersion().toByte();
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
index ef366b120..35ca580bc 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
@@ -14,8 +14,7 @@
  * during flush. The key is a fully qualified table name and the value is a set of channels that
  * belongs to this table
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
 class ChannelCache<T> {
   // Cache to hold all the valid channels, the key for the outer map is FullyQualifiedTableName and
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index de7185009..881a4c2f3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -674,17 +674,6 @@ static int validateAndParseBoolean(Object input) {
         input.getClass(), "BOOLEAN", new String[] {"boolean", "Number", "String"});
   }
 
-  static void checkValueInRange(BigDecimal bigDecimalValue, int scale, int precision) {
-    if (bigDecimalValue.abs().compareTo(BigDecimal.TEN.pow(precision - scale)) >= 0) {
-      throw new SFException(
-          ErrorCode.INVALID_ROW,
-          bigDecimalValue,
-          String.format(
-              "Number out of representable exclusive range of (-1e%s..1e%s)",
-              precision - scale, precision - scale));
-    }
-  }
-
   static Set<String> allowedBooleanStringsLowerCased =
       Sets.newHashSet("1", "0", "yes", "no", "y", "n", "t", "f", "true", "false", "on", "off");
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 0ef790c6f..df5522469 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -58,8 +58,7 @@
  * <li>upload the blob to stage
  * <li>register the blob to the targeted Snowflake table
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
 class FlushService<T> {
 
@@ -468,6 +467,7 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
         // We need to maintain IV as a block counter for the whole file, even interleaved,
         // to align with decryption on the Snowflake query path.
         // TODO: address alignment for the header SNOW-557866
+        // TODO: encryption is not yet supported by server side for Parquet
         long iv = curDataSize / Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES;
         byte[] encryptedCompressedChunkData =
             Cryptor.encrypt(
@@ -559,8 +559,7 @@ BlobMetadata upload(String filePath, byte[] blob, List<ChunkMetadata> metadata)
         blob.length,
         System.currentTimeMillis() - startTime);
 
-    return BlobMetadata.createBlobMetadata(
-        filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata);
+    return new BlobMetadata(filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
index 37ab4afc4..4ac7acf8c 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
@@ -13,8 +13,7 @@
  * Interface to convert {@link ChannelData} buffered in {@link RowBuffer} to the underlying format
  * implementation for faster processing.
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
 public interface Flusher<T> {
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
deleted file mode 100644
index 0c7b12ff2..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-import java.util.List;
-import java.util.Map;
-
-/** Parquet data holder to buffer rows. */
-public class ParquetChunkData {
-  final List<List<Object>> rows;
-  final Map<String, String> metadata;
-
-  /**
-   * Construct parquet data chunk.
-   *
-   * @param rows chunk row set
-   * @param metadata chunk metadata
-   */
-  public ParquetChunkData(List<List<Object>> rows, Map<String, String> metadata) {
-    this.rows = rows;
-    this.metadata = metadata;
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
deleted file mode 100644
index 3d9827b6b..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-import static net.snowflake.ingest.utils.Constants.MAX_CHUNK_SIZE_IN_BYTES;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.SFException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.parquet.column.ColumnDescriptor;
-import org.apache.parquet.column.ParquetProperties;
-import org.apache.parquet.hadoop.ParquetFileWriter;
-import org.apache.parquet.hadoop.ParquetWriter;
-import org.apache.parquet.hadoop.api.WriteSupport;
-import org.apache.parquet.hadoop.metadata.CompressionCodecName;
-import org.apache.parquet.io.DelegatingPositionOutputStream;
-import org.apache.parquet.io.OutputFile;
-import org.apache.parquet.io.ParquetEncodingException;
-import org.apache.parquet.io.PositionOutputStream;
-import org.apache.parquet.io.api.Binary;
-import org.apache.parquet.io.api.RecordConsumer;
-import org.apache.parquet.schema.MessageType;
-import org.apache.parquet.schema.PrimitiveType;
-
-/**
- * Converts {@link ChannelData} buffered in {@link RowBuffer} to the Parquet format for faster
- * processing.
- */
-public class ParquetFlusher implements Flusher<ParquetChunkData> {
-  private static final Logging logger = new Logging(ParquetFlusher.class);
-  private final MessageType schema;
-
-  /** Construct parquet flusher from its schema. */
-  public ParquetFlusher(MessageType schema) {
-    this.schema = schema;
-  }
-
-  @Override
-  public SerializationResult serialize(
-      List<ChannelData<ParquetChunkData>> channelsDataPerTable,
-      ByteArrayOutputStream chunkData,
-      String filePath)
-      throws IOException {
-    List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
-    long rowCount = 0L;
-    SnowflakeStreamingIngestChannelInternal<ParquetChunkData> firstChannel = null;
-    Map<String, RowBufferStats> columnEpStatsMapCombined = null;
-    List<List<Object>> rows = null;
-
-    for (ChannelData<ParquetChunkData> data : channelsDataPerTable) {
-      // Create channel metadata
-      ChannelMetadata channelMetadata =
-          ChannelMetadata.builder()
-              .setOwningChannel(data.getChannel())
-              .setRowSequencer(data.getRowSequencer())
-              .setOffsetToken(data.getOffsetToken())
-              .build();
-      // Add channel metadata to the metadata list
-      channelsMetadataList.add(channelMetadata);
-
-      logger.logDebug(
-          "Parquet Flusher: Start building channel={}, rowCount={}, bufferSize={} in blob={}",
-          data.getChannel().getFullyQualifiedName(),
-          data.getRowCount(),
-          data.getBufferSize(),
-          filePath);
-
-      if (rows == null) {
-        columnEpStatsMapCombined = data.getColumnEps();
-        rows = new ArrayList<>();
-        firstChannel = data.getChannel();
-      } else {
-        // This method assumes that channelsDataPerTable is grouped by table. We double check
-        // here and throw an error if the assumption is violated
-        if (!data.getChannel()
-            .getFullyQualifiedTableName()
-            .equals(firstChannel.getFullyQualifiedTableName())) {
-          throw new SFException(ErrorCode.INVALID_DATA_IN_CHUNK);
-        }
-
-        columnEpStatsMapCombined =
-            ChannelData.getCombinedColumnStatsMap(columnEpStatsMapCombined, data.getColumnEps());
-      }
-      rows.addAll(data.getVectors().rows);
-
-      rowCount += data.getRowCount();
-
-      logger.logDebug(
-          "Parquet Flusher: Finish building channel={}, rowCount={}, bufferSize={} in blob={}",
-          data.getChannel().getFullyQualifiedName(),
-          data.getRowCount(),
-          data.getBufferSize(),
-          filePath);
-    }
-
-    Map<String, String> metadata = channelsDataPerTable.get(0).getVectors().metadata;
-
-    flushToParquetBdecChunk(chunkData, rows, metadata, channelsMetadataList);
-    return new SerializationResult(channelsMetadataList, columnEpStatsMapCombined, rowCount);
-  }
-
-  /**
-   * Flushes a parquet row chunk to the given BDEC output stream.
-   *
-   * @param bdecOutput BDEC output stream
-   * @param chunkRows chunk rows
-   * @param metadata chunk metadata
-   * @param channelsMetadataList metadata of the channels the chunk rows belong to
-   * @throws IOException thrown from Parquet library in case of writing problems
-   */
-  private void flushToParquetBdecChunk(
-      ByteArrayOutputStream bdecOutput,
-      List<List<Object>> chunkRows,
-      Map<String, String> metadata,
-      List<ChannelMetadata> channelsMetadataList)
-      throws IOException {
-    try {
-      ParquetWriter<List<Object>> writer =
-          new BdecParquetWriterBuilder(bdecOutput, schema, metadata, channelsMetadataList)
-              // PARQUET_2_0 uses Encoding.DELTA_BYTE_ARRAY for byte arrays (e.g. SF sb16)
-              // server side does not support it TODO: SNOW-657238
-              .withWriterVersion(ParquetProperties.WriterVersion.PARQUET_1_0)
-
-              // the dictionary encoding (Encoding.*_DICTIONARY) is not supported by server side
-              // scanner yet
-              .withDictionaryEncoding(false)
-
-              // Historically server side scanner supports only the case when the row number in all
-              // pages is the same.
-              // The quick fix is to effectively disable the page size/row limit
-              // to always have one page per chunk until server side is generalised.
-              .withPageSize((int) Constants.MAX_CHUNK_SIZE_IN_BYTES * 2)
-              .withPageRowCountLimit(chunkRows.size() + 1)
-              .enableValidation()
-              .withCompressionCodec(CompressionCodecName.GZIP)
-              .withWriteMode(ParquetFileWriter.Mode.CREATE)
-              .build();
-
-      // We can use lower level column writers and custom ValuesWriterFactory that uses plain byte
-      // array encoding used by PARQUET_1_0 and supported by server side
-      // TODO: SNOW-672143
-      for (List<Object> row : chunkRows) {
-        writer.write(row);
-      }
-      writer.close();
-    } catch (Throwable t) {
-      logger.logError("Parquet Flusher: failed to write", t);
-      throw t;
-    }
-  }
-
-  /**
-   * A parquet specific write builder.
-   *
-   * <p>This class is implemented as parquet library API requires, mostly to provide {@link
-   * BdecWriteSupport} implementation.
-   */
-  private static class BdecParquetWriterBuilder
-      extends ParquetWriter.Builder<List<Object>, BdecParquetWriterBuilder> {
-    private final MessageType schema;
-    private final Map<String, String> extraMetaData;
-    private final List<ChannelMetadata> channelsMetadataList;
-
-    protected BdecParquetWriterBuilder(
-        ByteArrayOutputStream stream,
-        MessageType schema,
-        Map<String, String> extraMetaData,
-        List<ChannelMetadata> channelsMetadataList) {
-      super(new ByteArrayOutputFile(stream));
-      this.schema = schema;
-      this.extraMetaData = extraMetaData;
-      this.channelsMetadataList = channelsMetadataList;
-    }
-
-    @Override
-    protected BdecParquetWriterBuilder self() {
-      return this;
-    }
-
-    @Override
-    protected WriteSupport<List<Object>> getWriteSupport(Configuration conf) {
-      return new BdecWriteSupport(schema, extraMetaData, channelsMetadataList);
-    }
-  }
-
-  /**
-   * A parquet specific file output implementation.
-   *
-   * <p>This class is implemented as parquet library API requires, mostly to create our {@link
-   * ByteArrayDelegatingPositionOutputStream} implementation.
-   */
-  private static class ByteArrayOutputFile implements OutputFile {
-    private final ByteArrayOutputStream stream;
-
-    private ByteArrayOutputFile(ByteArrayOutputStream stream) {
-      this.stream = stream;
-    }
-
-    @Override
-    public PositionOutputStream create(long blockSizeHint) throws IOException {
-      stream.reset();
-      return new ByteArrayDelegatingPositionOutputStream(stream);
-    }
-
-    @Override
-    public PositionOutputStream createOrOverwrite(long blockSizeHint) throws IOException {
-      return create(blockSizeHint);
-    }
-
-    @Override
-    public boolean supportsBlockSize() {
-      return false;
-    }
-
-    @Override
-    public long defaultBlockSize() {
-      return (int) MAX_CHUNK_SIZE_IN_BYTES;
-    }
-  }
-
-  /**
-   * A parquet specific output stream implementation.
-   *
-   * <p>This class is implemented as parquet library API requires, mostly to wrap our BDEC output
-   * {@link ByteArrayOutputStream}.
-   */
-  private static class ByteArrayDelegatingPositionOutputStream
-      extends DelegatingPositionOutputStream {
-    private final ByteArrayOutputStream stream;
-
-    public ByteArrayDelegatingPositionOutputStream(ByteArrayOutputStream stream) {
-      super(stream);
-      this.stream = stream;
-    }
-
-    @Override
-    public long getPos() {
-      return stream.size();
-    }
-  }
-
-  /**
-   * A parquet specific write support implementation.
-   *
-   * <p>This class is implemented as parquet library API requires, mostly to serialize user column
-   * values depending on type into Parquet {@link RecordConsumer} in {@link
-   * BdecWriteSupport#write(List)}.
-   */
-  private static class BdecWriteSupport extends WriteSupport<List<Object>> {
-    MessageType schema;
-    RecordConsumer recordConsumer;
-    Map<String, String> extraMetadata;
-    List<ChannelMetadata> channelsMetadataList;
-
-    // TODO SNOW-672156: support specifying encodings and compression
-    BdecWriteSupport(
-        MessageType schema,
-        Map<String, String> extraMetadata,
-        List<ChannelMetadata> channelsMetadataList) {
-      this.schema = schema;
-      this.extraMetadata = extraMetadata;
-      this.channelsMetadataList = channelsMetadataList;
-    }
-
-    @Override
-    public WriteContext init(Configuration config) {
-      return new WriteContext(schema, extraMetadata);
-    }
-
-    @Override
-    public void prepareForWrite(RecordConsumer recordConsumer) {
-      this.recordConsumer = recordConsumer;
-    }
-
-    @Override
-    public void write(List<Object> values) {
-      List<ColumnDescriptor> cols = schema.getColumns();
-      if (values.size() != cols.size()) {
-        List<String> channelNames =
-            this.channelsMetadataList.stream()
-                .map(ChannelMetadata::getChannelName)
-                .collect(Collectors.toList());
-        throw new ParquetEncodingException(
-            "Invalid input data in channels "
-                + channelNames
-                + ". Expecting "
-                + cols.size()
-                + " columns. Input had "
-                + values.size()
-                + " columns ("
-                + cols
-                + ") : "
-                + values);
-      }
-
-      recordConsumer.startMessage();
-      for (int i = 0; i < cols.size(); ++i) {
-        Object val = values.get(i);
-        // val.length() == 0 indicates a NULL value.
-        if (val != null) {
-          String fieldName = cols.get(i).getPath()[0];
-          recordConsumer.startField(fieldName, i);
-          PrimitiveType.PrimitiveTypeName typeName =
-              cols.get(i).getPrimitiveType().getPrimitiveTypeName();
-          switch (typeName) {
-            case BOOLEAN:
-              recordConsumer.addBoolean((boolean) val);
-              break;
-            case FLOAT:
-              recordConsumer.addFloat((float) val);
-              break;
-            case DOUBLE:
-              recordConsumer.addDouble((double) val);
-              break;
-            case INT32:
-              recordConsumer.addInteger((int) val);
-              break;
-            case INT64:
-              recordConsumer.addLong((long) val);
-              break;
-            case BINARY:
-              recordConsumer.addBinary(Binary.fromString((String) val));
-              break;
-            case FIXED_LEN_BYTE_ARRAY:
-              Binary binary = Binary.fromConstantByteArray((byte[]) val);
-              recordConsumer.addBinary(binary);
-              break;
-            default:
-              throw new ParquetEncodingException(
-                  "Unsupported column type: " + cols.get(i).getPrimitiveType());
-          }
-          recordConsumer.endField(fieldName, i);
-        }
-      }
-      recordConsumer.endMessage();
-    }
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
deleted file mode 100644
index de4b3eadc..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import org.apache.parquet.column.ColumnDescriptor;
-import org.apache.parquet.schema.MessageType;
-import org.apache.parquet.schema.PrimitiveType;
-import org.apache.parquet.schema.Type;
-
-/**
- * The buffer in the Streaming Ingest channel that holds the un-flushed rows, these rows will be
- * converted to Parquet format for faster processing
- */
-public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
-  private static final Logging logger = new Logging(ParquetRowBuffer.class);
-
-  private static final String PARQUET_MESSAGE_TYPE_NAME = "bdec";
-
-  private final Map<String, Pair<ColumnMetadata, Integer>> fieldIndex;
-  private final Map<String, String> metadata;
-  private final List<List<Object>> data;
-  private final List<List<Object>> tempData;
-
-  private MessageType schema;
-
-  /**
-   * Construct a ParquetRowBuffer object
-   *
-   * @param channel client channel
-   */
-  ParquetRowBuffer(SnowflakeStreamingIngestChannelInternal<ParquetChunkData> channel) {
-    super(channel);
-    fieldIndex = new HashMap<>();
-    metadata = new HashMap<>();
-    data = new ArrayList<>();
-    tempData = new ArrayList<>();
-  }
-
-  @Override
-  public void setupSchema(List<ColumnMetadata> columns) {
-    fieldIndex.clear();
-    metadata.clear();
-    List<Type> parquetTypes = new ArrayList<>();
-    // Snowflake column id that corresponds to the order in 'columns' received from server
-    // id is required to pack column metadata for the server scanner, e.g. decimal scale and
-    // precision
-    int id = 1;
-    for (ColumnMetadata column : columns) {
-      ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-          ParquetTypeGenerator.generateColumnParquetTypeInfo(column, id);
-      parquetTypes.add(typeInfo.getParquetType());
-      this.metadata.putAll(typeInfo.getMetadata());
-      fieldIndex.put(column.getName(), new Pair<>(column, parquetTypes.size() - 1));
-      if (!column.getNullable()) {
-        addNonNullableFieldName(column.getName());
-      }
-      this.statsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
-
-      if (this.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.ABORT) {
-        this.tempStatsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
-      }
-
-      id++;
-    }
-    schema = new MessageType(PARQUET_MESSAGE_TYPE_NAME, parquetTypes);
-  }
-
-  @Override
-  boolean hasColumn(String name) {
-    return fieldIndex.containsKey(name);
-  }
-
-  @Override
-  float addRow(
-      Map<String, Object> row,
-      int curRowIndex,
-      Map<String, RowBufferStats> statsMap,
-      Set<String> formattedInputColumnNames) {
-    return addRow(row, data, statsMap, formattedInputColumnNames);
-  }
-
-  @Override
-  float addTempRow(
-      Map<String, Object> row,
-      int curRowIndex,
-      Map<String, RowBufferStats> statsMap,
-      Set<String> formattedInputColumnNames) {
-    return addRow(row, tempData, statsMap, formattedInputColumnNames);
-  }
-
-  /**
-   * Adds a row to the parquet buffer.
-   *
-   * @param row row to add
-   * @param out internal buffer to add to
-   * @param statsMap column stats map
-   * @param inputColumnNames list of input column names after formatting
-   * @return row size
-   */
-  private float addRow(
-      Map<String, Object> row,
-      List<List<Object>> out,
-      Map<String, RowBufferStats> statsMap,
-      Set<String> inputColumnNames) {
-    Object[] indexedRow = new Object[fieldIndex.size()];
-    float size = 0F;
-    for (Map.Entry<String, Object> entry : row.entrySet()) {
-      String key = entry.getKey();
-      Object value = entry.getValue();
-      String columnName = formatColumnName(key);
-      int colIndex = fieldIndex.get(columnName).getSecond();
-      RowBufferStats stats = statsMap.get(columnName);
-      ColumnMetadata column = fieldIndex.get(columnName).getFirst();
-      ColumnDescriptor columnDescriptor = schema.getColumns().get(colIndex);
-      PrimitiveType.PrimitiveTypeName typeName =
-          columnDescriptor.getPrimitiveType().getPrimitiveTypeName();
-      ParquetValueParser.ParquetBufferValue valueWithSize =
-          ParquetValueParser.parseColumnValueToParquet(value, column, typeName, stats);
-      indexedRow[colIndex] = valueWithSize.getValue();
-      size += valueWithSize.getSize();
-    }
-    out.add(Arrays.asList(indexedRow));
-
-    for (String columnName : Sets.difference(this.fieldIndex.keySet(), inputColumnNames)) {
-      statsMap.get(columnName).incCurrentNullCount();
-    }
-    return size;
-  }
-
-  @Override
-  void moveTempRowsToActualBuffer(int tempRowCount) {
-    data.addAll(tempData);
-  }
-
-  @Override
-  void clearTempRows() {
-    tempData.clear();
-  }
-
-  @Override
-  boolean hasColumns() {
-    return !fieldIndex.isEmpty();
-  }
-
-  @Override
-  Optional<ParquetChunkData> getSnapshot() {
-    List<List<Object>> oldData = new ArrayList<>();
-    data.forEach(r -> oldData.add(new ArrayList<>(r)));
-    return oldData.isEmpty()
-        ? Optional.empty()
-        : Optional.of(new ParquetChunkData(oldData, metadata));
-  }
-
-  /** Used only for testing. */
-  @Override
-  Object getVectorValueAt(String column, int index) {
-    int colIndex = fieldIndex.get(column).getSecond();
-    Object value = data.get(index).get(colIndex);
-    ColumnMetadata columnMetadata = fieldIndex.get(column).getFirst();
-    String physicalTypeStr = columnMetadata.getPhysicalType();
-    ColumnPhysicalType physicalType = ColumnPhysicalType.valueOf(physicalTypeStr);
-    String logicalTypeStr = columnMetadata.getLogicalType();
-    ColumnLogicalType logicalType = ColumnLogicalType.valueOf(logicalTypeStr);
-    if (logicalType == ColumnLogicalType.FIXED) {
-      if (physicalType == ColumnPhysicalType.SB1) {
-        value = ((Integer) value).byteValue();
-      }
-      if (physicalType == ColumnPhysicalType.SB2) {
-        value = ((Integer) value).shortValue();
-      }
-      if (physicalType == ColumnPhysicalType.SB16) {
-        value = new BigDecimal(new BigInteger((byte[]) value), columnMetadata.getScale());
-      }
-    }
-    if (logicalType == ColumnLogicalType.BINARY && value != null) {
-      value = ((String) value).getBytes(StandardCharsets.UTF_8);
-    }
-    return value;
-  }
-
-  @Override
-  int getTempRowCount() {
-    return tempData.size();
-  }
-
-  @Override
-  void reset() {
-    super.reset();
-    data.clear();
-  }
-
-  @Override
-  public void close(String name) {
-    this.fieldIndex.clear();
-    logger.logInfo(
-        "Trying to close parquet buffer for channel={} from function={}",
-        this.owningChannel.getName(),
-        name);
-  }
-
-  @Override
-  public Flusher<ParquetChunkData> createFlusher(Constants.BdecVersion bdecVerion) {
-    return new ParquetFlusher(schema);
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
deleted file mode 100644
index 149ce766c..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/*
- * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.SFException;
-import org.apache.parquet.schema.LogicalTypeAnnotation;
-import org.apache.parquet.schema.PrimitiveType;
-import org.apache.parquet.schema.Type;
-import org.apache.parquet.schema.Types;
-
-/** Generates the Parquet types for the Snowflake's column types */
-public class ParquetTypeGenerator {
-
-  /**
-   * Util class that contains Parquet type and other metadata for that type needed by the Snowflake
-   * server side scanner
-   */
-  static class ParquetTypeInfo {
-    private Type parquetType;
-    private Map<String, String> metadata;
-
-    public Type getParquetType() {
-      return this.parquetType;
-    }
-
-    public Map<String, String> getMetadata() {
-      return this.metadata;
-    }
-
-    public void setParquetType(Type parquetType) {
-      this.parquetType = parquetType;
-    }
-
-    public void setMetadata(Map<String, String> metadata) {
-      this.metadata = metadata;
-    }
-  }
-
-  private static final Set<AbstractRowBuffer.ColumnPhysicalType> TIME_SUPPORTED_PHYSICAL_TYPES =
-      new HashSet<>(
-          Arrays.asList(
-              AbstractRowBuffer.ColumnPhysicalType.SB4, AbstractRowBuffer.ColumnPhysicalType.SB8));
-  private static final Set<AbstractRowBuffer.ColumnPhysicalType>
-      TIMESTAMP_SUPPORTED_PHYSICAL_TYPES =
-          new HashSet<>(
-              Arrays.asList(
-                  AbstractRowBuffer.ColumnPhysicalType.SB8,
-                  AbstractRowBuffer.ColumnPhysicalType.SB16));
-
-  /**
-   * Generate the column parquet type and metadata from the column metadata received from server
-   * side.
-   *
-   * @param column column metadata as received from server side
-   * @param id column id
-   * @return column parquet type
-   */
-  static ParquetTypeInfo generateColumnParquetTypeInfo(ColumnMetadata column, int id) {
-    ParquetTypeInfo res = new ParquetTypeInfo();
-    Type parquetType;
-    Map<String, String> metadata = new HashMap<>();
-    String name = column.getName();
-
-    AbstractRowBuffer.ColumnPhysicalType physicalType;
-    AbstractRowBuffer.ColumnLogicalType logicalType;
-    try {
-      physicalType = AbstractRowBuffer.ColumnPhysicalType.valueOf(column.getPhysicalType());
-      logicalType = AbstractRowBuffer.ColumnLogicalType.valueOf(column.getLogicalType());
-    } catch (IllegalArgumentException e) {
-      throw new SFException(
-          ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
-    }
-
-    metadata.put(Integer.toString(id), logicalType.getOrdinal() + "," + physicalType.getOrdinal());
-
-    // Parquet Type.Repetition in general supports repeated values for the same row column, like a
-    // list of values.
-    // This generator uses only either 0 or 1 value for nullable data type (OPTIONAL: 0 or none
-    // value if it is null)
-    // or exactly 1 value for non-nullable data type (REQUIRED)
-    Type.Repetition repetition =
-        column.getNullable() ? Type.Repetition.OPTIONAL : Type.Repetition.REQUIRED;
-
-    // Handle differently depends on the column logical and physical types
-    switch (logicalType) {
-      case FIXED:
-        parquetType = getFixedColumnParquetType(column, id, physicalType, repetition);
-        break;
-      case ARRAY:
-      case OBJECT:
-      case VARIANT:
-        // mark the column metadata as being an object json for the server side scanner
-        metadata.put(id + ":obj_enc", "1");
-        // parquetType is same as the next one
-      case ANY:
-      case CHAR:
-      case TEXT:
-      case BINARY:
-        parquetType =
-            Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition)
-                .as(LogicalTypeAnnotation.stringType())
-                .id(id)
-                .named(name);
-        break;
-      case TIMESTAMP_LTZ:
-      case TIMESTAMP_NTZ:
-      case TIMESTAMP_TZ:
-        parquetType =
-            getTimeColumnParquetType(
-                column.getScale(),
-                physicalType,
-                logicalType,
-                TIMESTAMP_SUPPORTED_PHYSICAL_TYPES,
-                repetition,
-                id,
-                name);
-        break;
-      case DATE:
-        parquetType =
-            Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition)
-                .as(LogicalTypeAnnotation.dateType())
-                .id(id)
-                .named(name);
-        break;
-      case TIME:
-        parquetType =
-            getTimeColumnParquetType(
-                column.getScale(),
-                physicalType,
-                logicalType,
-                TIME_SUPPORTED_PHYSICAL_TYPES,
-                repetition,
-                id,
-                name);
-        break;
-      case BOOLEAN:
-        parquetType =
-            Types.primitive(PrimitiveType.PrimitiveTypeName.BOOLEAN, repetition).id(id).named(name);
-        break;
-      case REAL:
-        parquetType =
-            Types.primitive(PrimitiveType.PrimitiveTypeName.DOUBLE, repetition).id(id).named(name);
-        break;
-      default:
-        throw new SFException(
-            ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
-    }
-    res.setParquetType(parquetType);
-    res.setMetadata(metadata);
-    return res;
-  }
-
-  /**
-   * Get the parquet type for column of Snowflake FIXED logical type.
-   *
-   * @param column column metadata
-   * @param id column id in Snowflake table schema
-   * @param physicalType Snowflake physical type of column
-   * @param repetition parquet repetition type of column
-   * @return column parquet type
-   */
-  private static Type getFixedColumnParquetType(
-      ColumnMetadata column,
-      int id,
-      AbstractRowBuffer.ColumnPhysicalType physicalType,
-      Type.Repetition repetition) {
-    String name = column.getName();
-    // the LogicalTypeAnnotation.DecimalLogicalTypeAnnotation is used by server side scanner
-    // to discover data type scale and precision
-    LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalLogicalTypeAnnotation =
-        column.getScale() != null && column.getPrecision() != null
-            ? LogicalTypeAnnotation.DecimalLogicalTypeAnnotation.decimalType(
-                column.getScale(), column.getPrecision())
-            : null;
-    Type parquetType;
-    if ((column.getScale() != null && column.getScale() != 0)
-        || physicalType == AbstractRowBuffer.ColumnPhysicalType.SB16) {
-      parquetType =
-          Types.primitive(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, repetition)
-              .length(16)
-              .as(decimalLogicalTypeAnnotation)
-              .id(id)
-              .named(name);
-    } else {
-      switch (physicalType) {
-        case SB1:
-        case SB2:
-        case SB4:
-          parquetType =
-              Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition)
-                  .as(decimalLogicalTypeAnnotation)
-                  .id(id)
-                  .length(4)
-                  .named(name);
-          break;
-        case SB8:
-          parquetType =
-              Types.primitive(PrimitiveType.PrimitiveTypeName.INT64, repetition)
-                  .as(decimalLogicalTypeAnnotation)
-                  .id(id)
-                  .length(8)
-                  .named(name);
-          break;
-        default:
-          throw new SFException(
-              ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
-      }
-    }
-    return parquetType;
-  }
-
-  /**
-   * Get the parquet type for column of a Snowflake time logical type.
-   *
-   * @param scale column scale
-   * @param physicalType Snowflake physical type of column
-   * @param logicalType Snowflake logical type of column
-   * @param supportedPhysicalTypes supported Snowflake physical types for the given column
-   * @param repetition parquet repetition type of column
-   * @param id column id in Snowflake table schema
-   * @param name column name
-   * @return column parquet type
-   */
-  private static Type getTimeColumnParquetType(
-      Integer scale,
-      AbstractRowBuffer.ColumnPhysicalType physicalType,
-      AbstractRowBuffer.ColumnLogicalType logicalType,
-      Set<AbstractRowBuffer.ColumnPhysicalType> supportedPhysicalTypes,
-      Type.Repetition repetition,
-      int id,
-      String name) {
-    if (scale == null || scale > 9 || scale < 0 || !supportedPhysicalTypes.contains(physicalType)) {
-      throw new SFException(
-          ErrorCode.UNKNOWN_DATA_TYPE,
-          "Data type: " + logicalType + ", " + physicalType + ", scale: " + scale);
-    }
-
-    PrimitiveType.PrimitiveTypeName type = getTimePrimitiveType(physicalType);
-    LogicalTypeAnnotation typeAnnotation;
-    int length;
-    switch (physicalType) {
-      case SB4:
-        typeAnnotation = LogicalTypeAnnotation.decimalType(scale, 9);
-        length = 4;
-        break;
-      case SB8:
-        typeAnnotation = LogicalTypeAnnotation.decimalType(scale, 18);
-        length = 8;
-        break;
-      case SB16:
-        typeAnnotation = LogicalTypeAnnotation.decimalType(scale, 38);
-        length = 16;
-        break;
-      default:
-        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
-    }
-    return Types.primitive(type, repetition).as(typeAnnotation).length(length).id(id).named(name);
-  }
-
-  /**
-   * Get the parquet primitive type name for column of a Snowflake time logical type.
-   *
-   * @param physicalType Snowflake physical type of column
-   * @return column parquet primitive type name
-   */
-  private static PrimitiveType.PrimitiveTypeName getTimePrimitiveType(
-      AbstractRowBuffer.ColumnPhysicalType physicalType) {
-    PrimitiveType.PrimitiveTypeName type;
-    switch (physicalType) {
-      case SB4:
-        type = PrimitiveType.PrimitiveTypeName.INT32;
-        break;
-      case SB8:
-        type = PrimitiveType.PrimitiveTypeName.INT64;
-        break;
-      case SB16:
-        type = PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY;
-        break;
-      default:
-        throw new UnsupportedOperationException("Time physical type: " + physicalType);
-    }
-    return type;
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
deleted file mode 100644
index 671af1a5b..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.math.RoundingMode;
-import java.nio.charset.StandardCharsets;
-import java.util.Optional;
-import javax.annotation.Nullable;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.parquet.schema.PrimitiveType;
-
-/** Parses a user column value into Parquet internal representation for buffering. */
-class ParquetValueParser {
-  /** Parquet internal value representation for buffering. */
-  static class ParquetBufferValue {
-    private final Object value;
-    private final float size;
-
-    ParquetBufferValue(Object value, float size) {
-      this.value = value;
-      this.size = size;
-    }
-
-    Object getValue() {
-      return value;
-    }
-
-    float getSize() {
-      return size;
-    }
-  }
-
-  /**
-   * Parses a user column value into Parquet internal representation for buffering.
-   *
-   * @param value column value provided by user in a row
-   * @param columnMetadata column metadata
-   * @param typeName Parquet primitive type name
-   * @param stats column stats to update
-   * @return parsed value and byte size of Parquet internal representation
-   */
-  static ParquetBufferValue parseColumnValueToParquet(
-      Object value,
-      ColumnMetadata columnMetadata,
-      PrimitiveType.PrimitiveTypeName typeName,
-      RowBufferStats stats) {
-    Utils.assertNotNull("Parquet column stats", stats);
-    float size = 0F;
-    if (value != null) {
-      AbstractRowBuffer.ColumnLogicalType logicalType =
-          AbstractRowBuffer.ColumnLogicalType.valueOf(columnMetadata.getLogicalType());
-      AbstractRowBuffer.ColumnPhysicalType physicalType =
-          AbstractRowBuffer.ColumnPhysicalType.valueOf(columnMetadata.getPhysicalType());
-      switch (typeName) {
-        case BOOLEAN:
-          int intValue = DataValidationUtil.validateAndParseBoolean(value);
-          value = intValue > 0;
-          stats.addIntValue(BigInteger.valueOf(intValue));
-          size = 1;
-          break;
-        case INT32:
-          int intVal =
-              getInt32Value(
-                  value,
-                  columnMetadata.getScale(),
-                  Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
-                  logicalType,
-                  physicalType);
-          value = intVal;
-          stats.addIntValue(BigInteger.valueOf(intVal));
-          size = 4;
-          break;
-        case INT64:
-          long longValue =
-              getInt64Value(
-                  value,
-                  columnMetadata.getScale(),
-                  Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
-                  logicalType,
-                  physicalType);
-          value = longValue;
-          stats.addIntValue(BigInteger.valueOf(longValue));
-          size = 8;
-          break;
-        case DOUBLE:
-          double doubleValue = DataValidationUtil.validateAndParseReal(value);
-          value = doubleValue;
-          stats.addRealValue(doubleValue);
-          size = 8;
-          break;
-        case BINARY:
-          String str = getBinaryValue(value, stats, columnMetadata);
-          value = str;
-          size = str.getBytes().length;
-          break;
-        case FIXED_LEN_BYTE_ARRAY:
-          BigInteger intRep =
-              getSb16Value(
-                  value,
-                  columnMetadata.getScale(),
-                  Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
-                  logicalType,
-                  physicalType);
-          stats.addIntValue(intRep);
-          value = getSb16Bytes(intRep);
-          size += 16;
-          break;
-        default:
-          throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
-      }
-    } else {
-      if (!columnMetadata.getNullable()) {
-        throw new SFException(
-            ErrorCode.INVALID_ROW, columnMetadata.getName(), "Passed null to non nullable field");
-      }
-      stats.incCurrentNullCount();
-    }
-    return new ParquetBufferValue(value, size);
-  }
-
-  /**
-   * Parses an int32 value based on Snowflake logical type.
-   *
-   * @param value column value provided by user in a row
-   * @param scale data type scale
-   * @param precision data type precision
-   * @param logicalType Snowflake logical type
-   * @param physicalType Snowflake physical type
-   * @return parsed int32 value
-   */
-  private static int getInt32Value(
-      Object value,
-      @Nullable Integer scale,
-      Integer precision,
-      AbstractRowBuffer.ColumnLogicalType logicalType,
-      AbstractRowBuffer.ColumnPhysicalType physicalType) {
-    int intVal;
-    switch (logicalType) {
-      case DATE:
-        intVal = DataValidationUtil.validateAndParseDate(value);
-        break;
-      case TIME:
-        Utils.assertNotNull("Unexpected null scale for TIME data type", scale);
-        intVal = DataValidationUtil.validateAndParseTime(value, scale).intValue();
-        break;
-      case FIXED:
-        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
-        bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
-        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
-        intVal = bigDecimalValue.intValue();
-        break;
-      default:
-        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
-    }
-    return intVal;
-  }
-
-  /**
-   * Parses an int64 value based on Snowflake logical type.
-   *
-   * @param value column value provided by user in a row
-   * @param scale data type scale
-   * @param precision data type precision
-   * @param logicalType Snowflake logical type
-   * @param physicalType Snowflake physical type
-   * @return parsed int64 value
-   */
-  private static long getInt64Value(
-      Object value,
-      int scale,
-      int precision,
-      AbstractRowBuffer.ColumnLogicalType logicalType,
-      AbstractRowBuffer.ColumnPhysicalType physicalType) {
-    long longValue;
-    switch (logicalType) {
-      case TIME:
-        Utils.assertNotNull("Unexpected null scale for TIME data type", scale);
-        longValue = DataValidationUtil.validateAndParseTime(value, scale).longValue();
-        break;
-      case TIMESTAMP_LTZ:
-      case TIMESTAMP_NTZ:
-        boolean ignoreTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
-        longValue =
-            DataValidationUtil.validateAndParseTimestampNtzSb16(value, scale, ignoreTimezone)
-                .getTimeInScale()
-                .longValue();
-        break;
-      case TIMESTAMP_TZ:
-        longValue =
-            DataValidationUtil.validateAndParseTimestampTz(value, scale)
-                .getSfTimestamp()
-                .orElseThrow(
-                    () ->
-                        new SFException(
-                            ErrorCode.INVALID_ROW,
-                            value,
-                            "Unable to parse timestamp for TIMESTAMP_TZ column"))
-                .toBinary(scale, true)
-                .longValue();
-        break;
-      case FIXED:
-        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
-        bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
-        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
-        longValue = bigDecimalValue.longValue();
-        break;
-      default:
-        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
-    }
-    return longValue;
-  }
-
-  /**
-   * Parses an int128 value based on Snowflake logical type.
-   *
-   * @param value column value provided by user in a row
-   * @param scale data type scale
-   * @param precision data type precision
-   * @param logicalType Snowflake logical type
-   * @param physicalType Snowflake physical type
-   * @return parsed int64 value
-   */
-  private static BigInteger getSb16Value(
-      Object value,
-      int scale,
-      int precision,
-      AbstractRowBuffer.ColumnLogicalType logicalType,
-      AbstractRowBuffer.ColumnPhysicalType physicalType) {
-    switch (logicalType) {
-      case TIMESTAMP_TZ:
-        return DataValidationUtil.validateAndParseTimestampTz(value, scale)
-            .getSfTimestamp()
-            .orElseThrow(
-                () ->
-                    new SFException(
-                        ErrorCode.INVALID_ROW,
-                        value,
-                        "Unable to parse timestamp for TIMESTAMP_TZ column"))
-            .toBinary(scale, true);
-      case TIMESTAMP_LTZ:
-      case TIMESTAMP_NTZ:
-        boolean ignoreTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
-        return DataValidationUtil.validateAndParseTimestampNtzSb16(value, scale, ignoreTimezone)
-            .getTimeInScale();
-      case FIXED:
-        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
-        // explicitly match the BigDecimal input scale with the Snowflake data type scale
-        bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
-        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
-        return bigDecimalValue.unscaledValue();
-      default:
-        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
-    }
-  }
-
-  /**
-   * Converts an int128 value to its byte array representation.
-   *
-   * @param intRep int128 value
-   * @return byte array representation
-   */
-  static byte[] getSb16Bytes(BigInteger intRep) {
-    byte[] bytes = intRep.toByteArray();
-    byte padByte = (byte) (bytes[0] < 0 ? -1 : 0);
-    byte[] bytesBE = new byte[16];
-    for (int i = 0; i < 16 - bytes.length; i++) {
-      bytesBE[i] = padByte;
-    }
-    System.arraycopy(bytes, 0, bytesBE, 16 - bytes.length, bytes.length);
-    return bytesBE;
-  }
-
-  /**
-   * Converts an object, string or binary value to its byte array representation.
-   *
-   * @param value value to parse
-   * @param stats column stats to update
-   * @param columnMetadata column metadata
-   * @return string (byte array) representation
-   */
-  private static String getBinaryValue(
-      Object value, RowBufferStats stats, ColumnMetadata columnMetadata) {
-    AbstractRowBuffer.ColumnLogicalType logicalType =
-        AbstractRowBuffer.ColumnLogicalType.valueOf(columnMetadata.getLogicalType());
-    String str;
-    if (logicalType.isObject()) {
-      switch (logicalType) {
-        case OBJECT:
-          str = DataValidationUtil.validateAndParseObject(value);
-          break;
-        case VARIANT:
-          str = DataValidationUtil.validateAndParseVariant(value);
-          break;
-        case ARRAY:
-          str = DataValidationUtil.validateAndParseArray(value);
-          break;
-        default:
-          throw new SFException(
-              ErrorCode.UNKNOWN_DATA_TYPE, logicalType, columnMetadata.getPhysicalType());
-      }
-    } else if (logicalType == AbstractRowBuffer.ColumnLogicalType.BINARY) {
-      String maxLengthString = columnMetadata.getLength().toString();
-      byte[] bytes =
-          DataValidationUtil.validateAndParseBinary(
-              value, Optional.of(maxLengthString).map(Integer::parseInt));
-      str = new String(bytes, StandardCharsets.UTF_8);
-      stats.addStrValue(str);
-    } else {
-      String maxLengthString = columnMetadata.getLength().toString();
-      str =
-          DataValidationUtil.validateAndParseString(
-              value, Optional.of(maxLengthString).map(Integer::parseInt));
-      stats.addStrValue(str);
-    }
-    return str;
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index 44b90987b..739ae5499 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -25,8 +25,7 @@
  * Register one or more blobs to the targeted Snowflake table, it will be done using the dedicated
  * thread in order to maintain ordering per channel
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
 class RegisterService<T> {
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
index 2aa9d9861..a229f4db3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
@@ -13,8 +13,7 @@
  * Interface for the buffer in the Streaming Ingest channel that holds the un-flushed rows, these
  * rows will be converted to the underlying format implementation for faster processing
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
 interface RowBuffer<T> {
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 7016b23f5..d2d64db29 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -30,8 +30,7 @@
 /**
  * The first version of implementation for SnowflakeStreamingIngestChannel
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
 class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIngestChannel {
 
@@ -149,20 +148,9 @@ private RowBuffer<T> createRowBuffer(Constants.BdecVersion bdecVersion) {
     // TODO: The circular dependency SnowflakeStreamingIngestChannelInternal <-> RowBuffer
     // (SNOW-657667)
     // can be probably reconsidered
-    switch (bdecVersion) {
-      case ONE:
-      case TWO:
-        //noinspection unchecked
-        return (RowBuffer<T>)
-            new ArrowRowBuffer((SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>) this);
-      case THREE:
-        //noinspection unchecked
-        return (RowBuffer<T>)
-            new ParquetRowBuffer((SnowflakeStreamingIngestChannelInternal<ParquetChunkData>) this);
-      default:
-        throw new SFException(
-            ErrorCode.INTERNAL_ERROR, "Unsupported BDEC format version: " + bdecVersion);
-    }
+    //noinspection unchecked
+    return (RowBuffer<T>)
+        new ArrowRowBuffer((SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>) this);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 8be3c4ed3..6a3e99382 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -82,8 +82,7 @@
  * <li>the channel cache, which contains all the channels that belong to this account
  * <li>the flush service, which schedules and coordinates the flush to Snowflake tables
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
  */
 public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStreamingIngestClient {
 
@@ -554,7 +553,7 @@ List<BlobMetadata> getRetryBlobs(
                   .collect(Collectors.toList());
           if (!relevantChunks.isEmpty()) {
             retryBlobs.add(
-                BlobMetadata.createBlobMetadata(
+                new BlobMetadata(
                     blobMetadata.getPath(),
                     blobMetadata.getMD5(),
                     blobMetadata.getVersion(),
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index 68602e0a9..fed4022e1 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -83,14 +83,7 @@ public enum BdecVersion {
     ONE(1),
 
     /** Uses Arrow to generate BDEC chunks with {@link ArrowBatchWriteMode#FILE}. */
-    TWO(2),
-
-    /**
-     * Uses Parquet to generate BDEC chunks with {@link
-     * net.snowflake.ingest.streaming.internal.ParquetRowBuffer} (page-level compression). This
-     * version is experimental and WIP at the moment.
-     */
-    THREE(3);
+    TWO(2);
 
     private final byte version;
 
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 9f586a04c..5e5e594cd 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -14,6 +14,7 @@
 import static net.snowflake.ingest.utils.Constants.USER;
 import static net.snowflake.ingest.utils.Constants.WAREHOUSE;
 import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION;
+import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -34,7 +35,6 @@
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
-import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.Utils;
 import org.apache.commons.codec.binary.Base64;
 import org.junit.Assert;
@@ -180,7 +180,7 @@ public static KeyPair getKeyPair() throws Exception {
     return keyPair;
   }
 
-  public static Properties getProperties(Constants.BdecVersion bdecVersion) throws Exception {
+  public static Properties getProperties() throws Exception {
     if (profile == null) {
       init();
     }
@@ -195,10 +195,16 @@ public static Properties getProperties(Constants.BdecVersion bdecVersion) throws
     props.put(PRIVATE_KEY, privateKeyPem);
     props.put(ROLE, role);
     props.put(ACCOUNT_URL, getAccountURL());
-    props.put(BLOB_FORMAT_VERSION, bdecVersion.toByte());
+    props.put(BLOB_FORMAT_VERSION, getBlobFormatVersion());
     return props;
   }
 
+  private static byte getBlobFormatVersion() {
+    return profile.has(BLOB_FORMAT_VERSION)
+        ? (byte) profile.get(BLOB_FORMAT_VERSION).asInt()
+        : BLOB_FORMAT_VERSION_DEFAULT.toByte();
+  }
+
   /**
    * Create snowflake jdbc connection
    *
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
index 5785bd221..c07966c68 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
@@ -57,13 +57,15 @@ public void testFieldNumberAfterFlush() {
   @Test
   public void buildFieldFixedSB1() {
     // FIXED, SB1
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB1")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB1");
+    testCol.setNullable(true);
+    testCol.setLogicalType("FIXED");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -78,13 +80,15 @@ public void buildFieldFixedSB1() {
 
   @Test
   public void buildFieldFixedSB2() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB2")
-            .nullable(false)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB2");
+    testCol.setNullable(false);
+    testCol.setLogicalType("FIXED");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -99,13 +103,15 @@ public void buildFieldFixedSB2() {
 
   @Test
   public void buildFieldFixedSB4() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB4")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB4");
+    testCol.setNullable(true);
+    testCol.setLogicalType("FIXED");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -119,13 +125,15 @@ public void buildFieldFixedSB4() {
 
   @Test
   public void buildFieldFixedSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB8");
+    testCol.setNullable(true);
+    testCol.setLogicalType("FIXED");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -139,13 +147,15 @@ public void buildFieldFixedSB8() {
 
   @Test
   public void buildFieldFixedSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB16");
+    testCol.setNullable(true);
+    testCol.setLogicalType("FIXED");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     ArrowType expectedType =
@@ -162,13 +172,15 @@ public void buildFieldFixedSB16() {
 
   @Test
   public void buildFieldLobVariant() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("VARIANT")
-            .physicalType("LOB")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("LOB");
+    testCol.setNullable(true);
+    testCol.setLogicalType("VARIANT");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -182,13 +194,15 @@ public void buildFieldLobVariant() {
 
   @Test
   public void buildFieldTimestampNtzSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_NTZ")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB8");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIMESTAMP_NTZ");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -202,13 +216,15 @@ public void buildFieldTimestampNtzSB8() {
 
   @Test
   public void buildFieldTimestampNtzSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_NTZ")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB16");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIMESTAMP_NTZ");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -226,13 +242,15 @@ public void buildFieldTimestampNtzSB16() {
 
   @Test
   public void buildFieldTimestampTzSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_TZ")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB8");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIMESTAMP_TZ");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -250,13 +268,15 @@ public void buildFieldTimestampTzSB8() {
 
   @Test
   public void buildFieldTimestampTzSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_TZ")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB16");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIMESTAMP_TZ");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -275,14 +295,16 @@ public void buildFieldTimestampTzSB16() {
   }
 
   @Test
-  public void buildFieldTimestampDate() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("DATE")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
+  public void buildFieldDate() {
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB8");
+    testCol.setNullable(true);
+    testCol.setLogicalType("DATE");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -296,13 +318,15 @@ public void buildFieldTimestampDate() {
 
   @Test
   public void buildFieldTimeSB4() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIME")
-            .physicalType("SB4")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB4");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIME");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -316,13 +340,15 @@ public void buildFieldTimeSB4() {
 
   @Test
   public void buildFieldTimeSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIME")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB8");
+    testCol.setNullable(true);
+    testCol.setLogicalType("TIME");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -336,13 +362,15 @@ public void buildFieldTimeSB8() {
 
   @Test
   public void buildFieldBoolean() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("BOOLEAN")
-            .physicalType("BINARY")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("BINARY");
+    testCol.setNullable(true);
+    testCol.setLogicalType("BOOLEAN");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -356,13 +384,15 @@ public void buildFieldBoolean() {
 
   @Test
   public void buildFieldRealSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("REAL")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
+    ColumnMetadata testCol = new ColumnMetadata();
+    testCol.setName("testCol");
+    testCol.setPhysicalType("SB16");
+    testCol.setNullable(true);
+    testCol.setLogicalType("REAL");
+    testCol.setByteLength(14);
+    testCol.setLength(11);
+    testCol.setScale(0);
+    testCol.setPrecision(4);
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java b/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
deleted file mode 100644
index 51d4226bb..000000000
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-/** Helper class to build ColumnMetadata */
-public class ColumnMetadataBuilder {
-  private String name;
-  private String type;
-  private String logicalType;
-  private String physicalType;
-  private Integer precision;
-  private Integer scale;
-  private Integer byteLength;
-  private Integer length;
-  private boolean nullable;
-  private String collation;
-
-  /**
-   * Returns a new ColumnMetadata builder
-   *
-   * @return builder
-   */
-  public static ColumnMetadataBuilder newBuilder() {
-    ColumnMetadataBuilder mb = new ColumnMetadataBuilder();
-    mb.name = "testCol";
-    mb.byteLength = 14;
-    mb.length = 11;
-    mb.scale = 0;
-    mb.precision = 4;
-    return mb;
-  }
-
-  /**
-   * Set name
-   *
-   * @param name column name
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder name(String name) {
-    this.name = name;
-    return this;
-  }
-
-  /**
-   * Set type
-   *
-   * @param type type (as defined in ColumnMetadata)
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder type(String type) {
-    this.type = type;
-    return this;
-  }
-
-  /**
-   * Set column logical type
-   *
-   * @param logicalType logical type
-   * @return columnMetadata object
-   */
-  public ColumnMetadataBuilder logicalType(String logicalType) {
-    this.logicalType = logicalType;
-    return this;
-  }
-
-  /**
-   * Set column physical type
-   *
-   * @param physicalType physical type
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder physicalType(String physicalType) {
-    this.physicalType = physicalType;
-    return this;
-  }
-
-  /**
-   * Set column precision
-   *
-   * @param precision precision
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder precision(int precision) {
-    this.precision = precision;
-    return this;
-  }
-
-  /**
-   * Set column scale
-   *
-   * @param scale scale
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder scale(int scale) {
-    this.scale = scale;
-    return this;
-  }
-
-  /**
-   * Set column length in bytes
-   *
-   * @param byteLength length in bytes
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder byteLength(int byteLength) {
-    this.byteLength = byteLength;
-    return this;
-  }
-
-  /**
-   * Set column length (in chars)
-   *
-   * @param length length
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder length(int length) {
-    this.length = length;
-    return this;
-  }
-
-  /**
-   * Set column nullability
-   *
-   * @param nullable nullable
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder nullable(boolean nullable) {
-    this.nullable = nullable;
-    return this;
-  }
-
-  /**
-   * Set column collation
-   *
-   * @param collation collation
-   * @return columnMetadataBuilder object
-   */
-  public ColumnMetadataBuilder collation(String collation) {
-    this.collation = collation;
-    return this;
-  }
-
-  /**
-   * Build a columnMetadata object from the set values
-   *
-   * @return columnMetadata object
-   */
-  public ColumnMetadata build() {
-    ColumnMetadata colMetadata = new ColumnMetadata();
-    colMetadata.setName(name);
-    colMetadata.setType(type);
-    colMetadata.setPhysicalType(physicalType);
-    colMetadata.setNullable(nullable);
-    colMetadata.setLogicalType(logicalType);
-    colMetadata.setByteLength(byteLength);
-    colMetadata.setLength(length);
-    colMetadata.setScale(scale);
-    colMetadata.setPrecision(precision);
-    colMetadata.setCollation(collation);
-    return colMetadata;
-  }
-}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 540cd5ea8..80103e2df 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -55,8 +55,7 @@
 public class FlushServiceTest {
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> testContextFactory() {
-    return Arrays.asList(
-        new Object[][] {{ArrowTestContext.createFactory()}, {ParquetTestContext.createFactory()}});
+    return Arrays.asList(new Object[][] {{ArrowTestContext.createFactory()}});
   }
 
   public FlushServiceTest(TestContextFactory<?> testContextFactory) {
@@ -183,6 +182,11 @@ ChannelBuilder setEncryptionKeyId(Long encryptionKeyId) {
         return this;
       }
 
+      ChannelBuilder setOnErrorOption(OpenChannelRequest.OnErrorOption onErrorOption) {
+        this.onErrorOption = onErrorOption;
+        return this;
+      }
+
       SnowflakeStreamingIngestChannelInternal<T> buildAndAdd() {
         SnowflakeStreamingIngestChannelInternal<T> channel =
             createChannel(
@@ -281,48 +285,6 @@ TestContext<VectorSchemaRoot> create() {
     }
   }
 
-  private static class ParquetTestContext extends TestContext<List<List<Object>>> {
-
-    SnowflakeStreamingIngestChannelInternal<List<List<Object>>> createChannel(
-        String name,
-        String dbName,
-        String schemaName,
-        String tableName,
-        String offsetToken,
-        Long channelSequencer,
-        Long rowSequencer,
-        String encryptionKey,
-        Long encryptionKeyId,
-        OpenChannelRequest.OnErrorOption onErrorOption) {
-      return new SnowflakeStreamingIngestChannelInternal<>(
-          name,
-          dbName,
-          schemaName,
-          tableName,
-          offsetToken,
-          channelSequencer,
-          rowSequencer,
-          client,
-          encryptionKey,
-          encryptionKeyId,
-          onErrorOption,
-          Constants.BdecVersion.THREE,
-          null);
-    }
-
-    @Override
-    public void close() {}
-
-    static TestContextFactory<List<List<Object>>> createFactory() {
-      return new TestContextFactory<List<List<Object>>>("Parquet") {
-        @Override
-        TestContext<List<List<Object>>> create() {
-          return new ParquetTestContext();
-        }
-      };
-    }
-  }
-
   TestContextFactory<?> testContextFactory;
   TestContext<?> testContext;
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
deleted file mode 100644
index a477ac977..000000000
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
+++ /dev/null
@@ -1,480 +0,0 @@
-package net.snowflake.ingest.streaming.internal;
-
-import java.util.Map;
-import org.apache.parquet.schema.LogicalTypeAnnotation;
-import org.apache.parquet.schema.PrimitiveType;
-import org.apache.parquet.schema.Type;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class ParquetTypeGeneratorTest {
-
-  @Test
-  public void buildFieldFixedSB1() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB1")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(4)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
-        .expectedLogicalTypeAnnotation(
-            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB1.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldFixedSB2() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB2")
-            .nullable(false)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(4)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
-        .expectedLogicalTypeAnnotation(
-            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
-        .expectedRepetition(Type.Repetition.REQUIRED)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB2.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldFixedSB4() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB4")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(4)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
-        .expectedLogicalTypeAnnotation(
-            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB4.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldFixedSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(8)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
-        .expectedLogicalTypeAnnotation(
-            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldFixedSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(16)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
-        .expectedLogicalTypeAnnotation(
-            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldLobVariant() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("VARIANT")
-            .physicalType("LOB")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(0)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.BINARY)
-        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.stringType())
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.VARIANT.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.LOB.getOrdinal())
-        .assertMatches();
-
-    Assert.assertEquals("1", typeInfo.getMetadata().get(0 + ":obj_enc"));
-  }
-
-  @Test
-  public void buildFieldTimestampNtzSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_NTZ")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(8)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
-        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldTimestampNtzSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_NTZ")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(16)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
-        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 38))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldTimestampTzSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_TZ")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(8)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
-        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_TZ.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldTimestampTzSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_TZ")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(16)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
-        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 38))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_TZ.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldDate() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("DATE")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(0)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
-        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.dateType())
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.DATE.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldTimeSB4() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIME")
-            .physicalType("SB4")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(4)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
-        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 9))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.TIME.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB4.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldTimeSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIME")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(8)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
-        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.TIME.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldBoolean() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("BOOLEAN")
-            .physicalType("BINARY")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(0)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.BOOLEAN)
-        .expectedLogicalTypeAnnotation(null)
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.BOOLEAN.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.BINARY.getOrdinal())
-        .assertMatches();
-  }
-
-  @Test
-  public void buildFieldRealSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("REAL")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
-    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
-        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
-        .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
-        .expectedFieldId(0)
-        .expectedTypeLength(0)
-        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.DOUBLE)
-        .expectedLogicalTypeAnnotation(null)
-        .expectedRepetition(Type.Repetition.OPTIONAL)
-        .expectedColMetaData(
-            AbstractRowBuffer.ColumnLogicalType.REAL.getOrdinal()
-                + ","
-                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
-        .assertMatches();
-  }
-
-  /** Builder that helps to assert parquet type info */
-  private static class ParquetTypeInfoAssertionBuilder {
-    private String fieldName;
-    private int fieldId;
-    private PrimitiveType.PrimitiveTypeName primitiveTypeName;
-    private LogicalTypeAnnotation logicalTypeAnnotation;
-    private Type.Repetition repetition;
-    private int typeLength;
-    private String colMetadata;
-    private ParquetTypeGenerator.ParquetTypeInfo typeInfo;
-
-    static ParquetTypeInfoAssertionBuilder newBuilder() {
-      ParquetTypeInfoAssertionBuilder builder = new ParquetTypeInfoAssertionBuilder();
-      return builder;
-    }
-
-    ParquetTypeInfoAssertionBuilder typeInfo(ParquetTypeGenerator.ParquetTypeInfo typeInfo) {
-      this.typeInfo = typeInfo;
-      return this;
-    }
-
-    ParquetTypeInfoAssertionBuilder expectedFieldName(String fieldName) {
-      this.fieldName = fieldName;
-      return this;
-    }
-
-    ParquetTypeInfoAssertionBuilder expectedFieldId(int fieldId) {
-      this.fieldId = fieldId;
-      return this;
-    }
-
-    ParquetTypeInfoAssertionBuilder expectedPrimitiveTypeName(
-        PrimitiveType.PrimitiveTypeName primitiveTypeName) {
-      this.primitiveTypeName = primitiveTypeName;
-      return this;
-    }
-
-    ParquetTypeInfoAssertionBuilder expectedLogicalTypeAnnotation(
-        LogicalTypeAnnotation logicalTypeAnnotation) {
-      this.logicalTypeAnnotation = logicalTypeAnnotation;
-      return this;
-    }
-
-    ParquetTypeInfoAssertionBuilder expectedRepetition(Type.Repetition repetition) {
-      this.repetition = repetition;
-      return this;
-    }
-
-    ParquetTypeInfoAssertionBuilder expectedTypeLength(int typeLength) {
-      this.typeLength = typeLength;
-      return this;
-    }
-
-    ParquetTypeInfoAssertionBuilder expectedColMetaData(String colMetaData) {
-      this.colMetadata = colMetaData;
-      return this;
-    }
-
-    void assertMatches() {
-      Type type = typeInfo.getParquetType();
-      Map<String, String> metadata = typeInfo.getMetadata();
-      Assert.assertEquals(fieldName, type.getName());
-      Assert.assertEquals(typeLength, type.asPrimitiveType().getTypeLength());
-      Assert.assertEquals(fieldId, type.asPrimitiveType().getId().intValue());
-
-      Assert.assertEquals(primitiveTypeName, type.asPrimitiveType().getPrimitiveTypeName());
-      Assert.assertEquals(logicalTypeAnnotation, type.getLogicalTypeAnnotation());
-      Assert.assertEquals(repetition, type.getRepetition());
-      Assert.assertEquals(colMetadata, metadata.get(type.getId().toString()));
-    }
-  }
-}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
deleted file mode 100644
index 3dc313dc6..000000000
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
+++ /dev/null
@@ -1,570 +0,0 @@
-package net.snowflake.ingest.streaming.internal;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.HashMap;
-import java.util.Map;
-import net.snowflake.ingest.utils.SFException;
-import org.apache.parquet.schema.PrimitiveType;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class ParquetValueParserTest {
-
-  @Test
-  public void parseValueFixedSB1ToInt32() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB1")
-            .scale(0)
-            .precision(2)
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            12, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Integer.class)
-        .expectedParsedValue(12)
-        .expectedSize(4.0f)
-        .expectedMinMax(BigInteger.valueOf(12))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueFixedSB2ToInt32() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB2")
-            .scale(0)
-            .precision(4)
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            1234, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Integer.class)
-        .expectedParsedValue(1234)
-        .expectedSize(4.0f)
-        .expectedMinMax(BigInteger.valueOf(1234))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueFixedSB4ToInt32() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB4")
-            .scale(0)
-            .precision(9)
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            123456789, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Integer.class)
-        .expectedParsedValue(123456789)
-        .expectedSize(4.0f)
-        .expectedMinMax(BigInteger.valueOf(123456789))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueFixedSB8ToInt64() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB8")
-            .scale(0)
-            .precision(18)
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            123456789987654321L, testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Long.class)
-        .expectedParsedValue(123456789987654321L)
-        .expectedSize(8.0f)
-        .expectedMinMax(BigInteger.valueOf(123456789987654321L))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueFixedSB16ToByteArray() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB16")
-            .scale(0)
-            .precision(38)
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            new BigDecimal("91234567899876543219876543211234567891"),
-            testCol,
-            PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
-            rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(byte[].class)
-        .expectedParsedValue(
-            ParquetValueParser.getSb16Bytes(
-                new BigInteger("91234567899876543219876543211234567891")))
-        .expectedSize(16.0f)
-        .expectedMinMax(new BigInteger("91234567899876543219876543211234567891"))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueFixedDecimalToInt32() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB8")
-            .scale(5)
-            .precision(10)
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            new BigDecimal("12345.54321"),
-            testCol,
-            PrimitiveType.PrimitiveTypeName.DOUBLE,
-            rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Double.class)
-        .expectedParsedValue(Double.valueOf("12345.54321"))
-        .expectedSize(8.0f)
-        .expectedMinMax(Double.valueOf("12345.54321"))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueDouble() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("REAL")
-            .physicalType("DOUBLE")
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            12345.54321d, testCol, PrimitiveType.PrimitiveTypeName.DOUBLE, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Double.class)
-        .expectedParsedValue(Double.valueOf(12345.54321))
-        .expectedSize(8.0f)
-        .expectedMinMax(Double.valueOf(12345.54321))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueBoolean() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("BOOLEAN")
-            .physicalType("SB1")
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            true, testCol, PrimitiveType.PrimitiveTypeName.BOOLEAN, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Boolean.class)
-        .expectedParsedValue(true)
-        .expectedSize(1.0f)
-        .expectedMinMax(BigInteger.valueOf(1))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueBinary() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("BINARY")
-            .physicalType("LOB")
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            "Length7".getBytes(), testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(String.class)
-        .expectedParsedValue("Length7")
-        .expectedSize(7.0f)
-        .expectedMinMax("Length7")
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueVariantToBinary() {
-    testJsonWithLogicalType("VARIANT");
-  }
-
-  @Test
-  public void parseValueObjectToBinary() {
-    testJsonWithLogicalType("OBJECT");
-  }
-
-  private void testJsonWithLogicalType(String logicalType) {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType(logicalType)
-            .physicalType("BINARY")
-            .nullable(true)
-            .build();
-
-    String var =
-        "{\"key1\":-879869596,\"key2\":\"value2\",\"key3\":null,"
-            + "\"key4\":{\"key41\":0.032437,\"key42\":\"value42\",\"key43\":null}}";
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            var, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(String.class)
-        .expectedParsedValue(var)
-        .expectedSize(var.getBytes().length)
-        .expectedMinMax(null)
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueArrayToBinary() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("ARRAY")
-            .physicalType("BINARY")
-            .nullable(true)
-            .build();
-
-    Map<String, String> input = new HashMap<>();
-    input.put("a", "1");
-    input.put("b", "2");
-    input.put("c", "3");
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            input, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
-
-    String resultArray = "[{\"a\":\"1\",\"b\":\"2\",\"c\":\"3\"}]";
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(String.class)
-        .expectedParsedValue(resultArray)
-        .expectedSize(resultArray.length())
-        .expectedMinMax(null)
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueTimestampNtzSB4Error() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_NTZ")
-            .physicalType("SB4")
-            .scale(0) // seconds
-            .precision(9)
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    SFException exception =
-        Assert.assertThrows(
-            SFException.class,
-            () ->
-                ParquetValueParser.parseColumnValueToParquet(
-                    "2013-04-28 20:57:00",
-                    testCol,
-                    PrimitiveType.PrimitiveTypeName.INT32,
-                    rowBufferStats));
-    Assert.assertEquals(
-        "Unknown data type for logical: TIMESTAMP_NTZ, physical: SB4.", exception.getMessage());
-  }
-
-  @Test
-  public void parseValueTimestampNtzSB8ToINT64() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_NTZ")
-            .physicalType("SB8")
-            .scale(3) // millis
-            .precision(18)
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            "2013-04-28 20:57:01.000",
-            testCol,
-            PrimitiveType.PrimitiveTypeName.INT64,
-            rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Long.class)
-        .expectedParsedValue(1367182621000L)
-        .expectedSize(8.0f)
-        .expectedMinMax(BigInteger.valueOf(1367182621000L))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueTimestampNtzSB16ToByteArray() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_NTZ")
-            .physicalType("SB16")
-            .nullable(true)
-            .scale(9) // nanos
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            "2022-09-18 22:05:07.123456789",
-            testCol,
-            PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
-            rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(byte[].class)
-        .expectedParsedValue(
-            ParquetValueParser.getSb16Bytes(BigInteger.valueOf(1663538707123456789L)))
-        .expectedSize(16.0f)
-        .expectedMinMax(BigInteger.valueOf(1663538707123456789L))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueDateToInt32() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("DATE")
-            .physicalType("SB4")
-            .scale(0) // seconds
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            "2021-01-01", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Integer.class)
-        .expectedParsedValue(Integer.valueOf(18628))
-        .expectedSize(4.0f)
-        .expectedMinMax(BigInteger.valueOf(18628))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueTimeSB4ToInt32() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIME")
-            .physicalType("SB4")
-            .scale(0) // seconds
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            "01:00:00", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Integer.class)
-        .expectedParsedValue(3600)
-        .expectedSize(4.0f)
-        .expectedMinMax(BigInteger.valueOf(3600))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueTimeSB8ToInt64() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIME")
-            .physicalType("SB8")
-            .scale(3) // milliseconds
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    ParquetValueParser.ParquetBufferValue pv =
-        ParquetValueParser.parseColumnValueToParquet(
-            "01:00:00.123", testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats);
-
-    ParquetValueParserAssertionBuilder.newBuilder()
-        .parquetBufferValue(pv)
-        .rowBufferStats(rowBufferStats)
-        .expectedValueClass(Long.class)
-        .expectedParsedValue(3600123L)
-        .expectedSize(8.0f)
-        .expectedMinMax(BigInteger.valueOf(3600123))
-        .assertMatches();
-  }
-
-  @Test
-  public void parseValueTimeSB16Error() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIME")
-            .physicalType("SB16")
-            .scale(9) // nanos
-            .nullable(true)
-            .build();
-
-    RowBufferStats rowBufferStats = new RowBufferStats();
-    SFException exception =
-        Assert.assertThrows(
-            SFException.class,
-            () ->
-                ParquetValueParser.parseColumnValueToParquet(
-                    "11:00:00.12345678",
-                    testCol,
-                    PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
-                    rowBufferStats));
-    Assert.assertEquals(
-        "Unknown data type for logical: TIME, physical: SB16.", exception.getMessage());
-  }
-
-  /** Builder that helps to assert parsing of values to parquet types */
-  private static class ParquetValueParserAssertionBuilder {
-    private ParquetValueParser.ParquetBufferValue parquetBufferValue;
-    private RowBufferStats rowBufferStats;
-    private Class valueClass;
-    private Object value;
-    private float size;
-    private Object minMaxStat;
-
-    static ParquetValueParserAssertionBuilder newBuilder() {
-      ParquetValueParserAssertionBuilder builder = new ParquetValueParserAssertionBuilder();
-      return builder;
-    }
-
-    ParquetValueParserAssertionBuilder parquetBufferValue(
-        ParquetValueParser.ParquetBufferValue parquetBufferValue) {
-      this.parquetBufferValue = parquetBufferValue;
-      return this;
-    }
-
-    ParquetValueParserAssertionBuilder rowBufferStats(RowBufferStats rowBufferStats) {
-      this.rowBufferStats = rowBufferStats;
-      return this;
-    }
-
-    ParquetValueParserAssertionBuilder expectedValueClass(Class valueClass) {
-      this.valueClass = valueClass;
-      return this;
-    }
-
-    ParquetValueParserAssertionBuilder expectedParsedValue(Object value) {
-      this.value = value;
-      return this;
-    }
-
-    ParquetValueParserAssertionBuilder expectedSize(float size) {
-      this.size = size;
-      return this;
-    }
-
-    public ParquetValueParserAssertionBuilder expectedMinMax(Object minMaxStat) {
-      this.minMaxStat = minMaxStat;
-      return this;
-    }
-
-    void assertMatches() {
-      Assert.assertEquals(valueClass, parquetBufferValue.getValue().getClass());
-      System.out.println("parquetBufferValue = " + parquetBufferValue.getValue().toString());
-      if (valueClass.equals(byte[].class)) {
-        Assert.assertArrayEquals((byte[]) value, (byte[]) parquetBufferValue.getValue());
-      } else {
-        Assert.assertEquals(value, parquetBufferValue.getValue());
-      }
-      Assert.assertEquals(size, parquetBufferValue.getSize(), 0);
-      if (minMaxStat instanceof BigInteger) {
-        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinIntValue());
-        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxIntValue());
-        return;
-      } else if (minMaxStat instanceof String || valueClass.equals(String.class)) {
-        // String can have null min/max stats for variant data types
-        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinColStrValue());
-        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxColStrValue());
-        return;
-      } else if (minMaxStat instanceof Double || minMaxStat instanceof BigDecimal) {
-        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinRealValue());
-        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxRealValue());
-        return;
-      }
-      throw new IllegalArgumentException("Unknown data type for min stat");
-    }
-  }
-}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 8d28c0cb9..7bd986e8a 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -26,11 +26,7 @@
 public class RowBufferTest {
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> bdecVersion() {
-    return Arrays.asList(
-        new Object[][] {
-          {"Arrow", Constants.BdecVersion.ONE},
-          {"Parquet", Constants.BdecVersion.THREE}
-        });
+    return Arrays.asList(new Object[][] {{"Arrow", Constants.BdecVersion.ONE}});
   }
 
   private final Constants.BdecVersion bdecVersion;
@@ -1033,7 +1029,6 @@ private void testE2EBinaryHelper(OpenChannelRequest.OnErrorOption onErrorOption)
     colBinary.setPhysicalType("LOB");
     colBinary.setNullable(true);
     colBinary.setLogicalType("BINARY");
-    colBinary.setLength(32);
     colBinary.setScale(0);
 
     innerBuffer.setupSchema(Collections.singletonList(colBinary));
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index c5189bcde..175db5fb8 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -9,15 +9,14 @@
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Random;
 import java.util.TimeZone;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
@@ -32,7 +31,6 @@
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
-import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.ParameterProvider;
 import net.snowflake.ingest.utils.SFException;
@@ -40,23 +38,9 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 
 /** Example streaming ingest sdk integration test */
-@RunWith(Parameterized.class)
 public class StreamingIngestIT {
-  @Parameterized.Parameters(name = "{0}")
-  public static Collection<Object[]> bdecVersion() {
-    return Arrays.asList(
-        new Object[][] {
-          {"Arrow", Constants.BdecVersion.ONE},
-          // TODO: uncomment once SNOW-659721 is deployed and we set the parameter
-          // DISABLE_PARQUET_CACHE to true for the test account
-          // {"Parquet", Constants.BdecVersion.THREE}
-        });
-  }
-
   private static final String TEST_TABLE = "STREAMING_INGEST_TEST_TABLE";
   private static final String TEST_DB_PREFIX = "STREAMING_INGEST_TEST_DB";
   private static final String TEST_SCHEMA = "STREAMING_INGEST_TEST_SCHEMA";
@@ -71,23 +55,11 @@ public static Collection<Object[]> bdecVersion() {
   private Connection jdbcConnection;
   private String testDb;
 
-  private final Constants.BdecVersion bdecVersion;
-
-  public StreamingIngestIT(
-      @SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
-    this.bdecVersion = bdecVersion;
-  }
-
   @Before
   public void beforeAll() throws Exception {
     testDb = TEST_DB_PREFIX + "_" + UUID.randomUUID().toString().substring(0, 4);
     // Create a streaming ingest client
     jdbcConnection = TestUtils.getConnection(true);
-    if (bdecVersion == Constants.BdecVersion.THREE) {
-      // TODO: encryption and interleaved mode are not yet supported by server side's Parquet
-      // scanner if local file cache is enabled (SNOW-656500)
-      jdbcConnection.createStatement().execute("alter session set disable_parquet_cache=true;");
-    }
     jdbcConnection
         .createStatement()
         .execute(String.format("create or replace database %s;", testDb));
@@ -106,7 +78,7 @@ public void beforeAll() throws Exception {
         .createStatement()
         .execute(String.format("use warehouse %s", TestUtils.getWarehouse()));
 
-    prop = TestUtils.getProperties(bdecVersion);
+    prop = TestUtils.getProperties();
     if (prop.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       prop.setProperty(ROLE, "ACCOUNTADMIN");
     }
@@ -1070,7 +1042,7 @@ private void verifyTableRowCount(int rowNumber, String tableName) {
   }
 
   @Test
-  public void testDataTypes() throws SQLException, ParseException {
+  public void testDataTypes() {
     String tableName = "t_data_types";
     try {
       jdbcConnection
@@ -1108,195 +1080,94 @@ public void testDataTypes() throws SQLException, ParseException {
 
     SnowflakeStreamingIngestChannel channel = openChannel(tableName, "CHANNEL");
 
-    Map<String, Object> negRow = getNegRow();
-    verifyInsertValidationResponse(channel.insertRow(negRow, Integer.toString(0)));
-
-    Map<String, Object> nullRow = getNullRow();
-    verifyInsertValidationResponse(channel.insertRow(nullRow, Integer.toString(1)));
-
-    Map<String, Object> posRow = getPosRow();
-    verifyInsertValidationResponse(channel.insertRow(posRow, Integer.toString(2)));
-
-    waitChannelFlushed(channel, 3);
-    verifyTableRowCount(3, tableName);
-
-    ResultSet result =
-        jdbcConnection
-            .createStatement()
-            .executeQuery(
-                String.format(
-                    "select * from %s.%s.%s order by num", testDb, TEST_SCHEMA, tableName));
-
-    result.next(); // negRow
-    assertNegRow(negRow, result);
-
-    result.next(); // nullRow
-    assertNullRow(nullRow, result);
-
-    result.next(); // posRow
-    assertPosRow(posRow, result);
-  }
-
-  private Map<String, Object> getNegRow() {
-    Map<String, Object> negRow = new HashMap<>();
-    negRow.put("num", -123);
-    negRow.put("num_10_5", new BigDecimal("-12345.54321"));
-    negRow.put("num_38_0", new BigDecimal("-91234567899876543219876543211234567891"));
-    negRow.put("num_2_0", -12);
-    negRow.put("num_4_0", -1234);
-    negRow.put("num_9_0", -123456789);
-    negRow.put("num_18_0", -123456789987654321L);
-    negRow.put("num_float", -12345.54321f);
-    negRow.put("str_varchar", "zxccvbnm,.");
-    negRow.put("bin", new byte[] {1, 2, 3});
-    negRow.put("bl", true);
-    negRow.put("var", "{}");
-    negRow.put("obj", "{}");
-    negRow.put("arr", Arrays.asList());
-    negRow.put("epochdays", "0");
-    negRow.put("epochsec", "0");
-    negRow.put("epochnano", "0");
-    negRow.put("timesec", "0");
-    negRow.put("timenano", "0");
-
-    return negRow;
+    Random r = new Random();
+    for (int val = 0; val < 10000; val++) {
+      verifyInsertValidationResponse(channel.insertRow(getRandomRow(r), Integer.toString(val)));
+    }
+    waitChannelFlushed(channel, 10000);
   }
 
-  private Map<String, Object> getNullRow() {
-    Map<String, Object> nullRow = new HashMap<>();
-    nullRow.put("num", 0);
-    nullRow.put("num_10_5", new BigDecimal("0.00000"));
-    nullRow.put("num_38_0", new BigDecimal("0"));
-    nullRow.put("num_2_0", 0);
-    nullRow.put("num_4_0", 0);
-    nullRow.put("num_9_0", 0);
-    nullRow.put("num_18_0", 0L);
-    nullRow.put("num_float", 0.0f);
-    nullRow.put("str_varchar", null);
-    nullRow.put("bin", null);
-    nullRow.put("bl", false);
-    nullRow.put("var", null);
-    nullRow.put("obj", null);
-    nullRow.put("arr", null);
-    nullRow.put("epochdays", null);
-    nullRow.put("epochsec", null);
-    nullRow.put("epochnano", null);
-    nullRow.put("timesec", null);
-    nullRow.put("timenano", null);
-
-    return nullRow;
+  private static Map<String, Object> getRandomRow(Random r) {
+    Map<String, Object> row = new HashMap<>();
+    row.put("num", r.nextInt());
+    row.put("num_10_5", nextFloat(r));
+    row.put(
+        "num_38_0",
+        new BigDecimal("" + nextLongOfPrecision(r, 18) + Math.abs(nextLongOfPrecision(r, 18))));
+    row.put("num_2_0", r.nextInt(100));
+    row.put("num_4_0", r.nextInt(10000));
+    row.put("num_9_0", r.nextInt(1000000000));
+    row.put("num_18_0", nextLongOfPrecision(r, 18));
+    row.put("num_float", nextFloat(r));
+    row.put("str_varchar", nextString(r));
+    row.put("bin", nextBytes(r));
+    row.put("bl", r.nextBoolean());
+    row.put("var", nextJson(r));
+    row.put("obj", nextJson(r));
+    row.put("arr", Arrays.asList(r.nextInt(100), r.nextInt(100), r.nextInt(100)));
+    row.put(
+        "epochdays",
+        String.valueOf(Math.abs(r.nextInt()) % (18963 * 24 * 60 * 60))); // DATE, 02.12.2021
+    row.put(
+        "timesec",
+        String.valueOf(
+            (r.nextInt(11) * 60 * 60 + r.nextInt(59) * 60 + r.nextInt(59)) * 10000
+                + r.nextInt(9999))); // TIME(4), 05:12:43.4536
+    row.put(
+        "timenano",
+        String.valueOf(
+            (14 * 60 * 60 + 26 * 60 + 34) * 1000000000L
+                + 437582643)); // TIME(9), 14:26:34.437582643
+    row.put(
+        "epochsec",
+        String.valueOf(
+            Math.abs(r.nextLong()) % 1638459438)); // TIMESTAMP_LTZ(0), 02.12.2021 15:37:18
+    row.put(
+        "epochnano",
+        String.format(
+            "%d%d",
+            Math.abs(r.nextInt()) % 1999999999,
+            100000000
+                + Math.abs(
+                    r.nextInt(899999999)))); // TIMESTAMP_LTZ(9), 18.05.2033 03:33:19.999999999
+    return row;
   }
 
-  private Map<String, Object> getPosRow() {
-    Map<String, Object> posRow = new HashMap<>();
-    posRow.put("num", 123);
-    posRow.put("num_10_5", new BigDecimal("12345.54321"));
-    posRow.put("num_38_0", new BigDecimal("91234567899876543219876543211234567891"));
-    posRow.put("num_2_0", 12);
-    posRow.put("num_4_0", 1234);
-    posRow.put("num_9_0", 123456789);
-    posRow.put("num_18_0", 123456789987654321L);
-    posRow.put("num_float", 12345.54321f);
-    posRow.put("str_varchar", "abcd123456.");
-    posRow.put("bin", new byte[] {4, 5, 6});
-    posRow.put("bl", true);
-    posRow.put(
-        "var",
-        "{ \"a\": 1, \"b\": \"qwerty\", \"c\": null, \"d\": { \"e\": 2, \"f\": \"asdf\", \"g\":"
-            + " null } }");
-    posRow.put(
-        "obj",
-        "{ \"a\": 1, \"b\": \"qwerty\", \"c\": null, \"d\": { \"e\": 2, \"f\": \"asdf\", \"g\":"
-            + " null } }");
-    posRow.put("arr", Arrays.asList("{ \"a\": 1}", "{ \"b\": 2 }", "{ \"c\": 3 }"));
-    posRow.put("epochdays", "2022-09-18 20:05:07"); // DATE, 18.09.2022
-    posRow.put("epochsec", "2022-09-18 20:05:07"); // TIMESTAMP_NTZ(0)
-    posRow.put("epochnano", "2022-09-18 20:05:07.999999999"); // TIMESTAMP_NTZ(9)
-    posRow.put("timesec", "01:00:01.999999999"); // TIME(0)
-    posRow.put("timenano", "01:00:01.999999999"); // TIME(9)
-
-    return posRow;
+  private static long nextLongOfPrecision(Random r, int precision) {
+    return r.nextLong() % Math.round(Math.pow(10, precision));
   }
 
-  private void assertNegRow(Map<String, Object> expectedNegRow, ResultSet actualResult)
-      throws SQLException {
-    assertNonTimeAndVarFields(expectedNegRow, actualResult);
-    Assert.assertEquals("{}", actualResult.getString("VAR"));
-    Assert.assertEquals("{}", actualResult.getString("OBJ"));
-    Assert.assertEquals("[]", actualResult.getString("ARR"));
-    Assert.assertEquals(0, actualResult.getDate("EPOCHDAYS").getTime());
-    Assert.assertEquals(0, actualResult.getTimestamp("EPOCHSEC").getTime());
-    Assert.assertEquals(0, actualResult.getTimestamp("EPOCHNANO").getNanos());
-    Assert.assertEquals(0, actualResult.getTimestamp("EPOCHNANO").getTime());
-    Assert.assertEquals(0, actualResult.getTimestamp("TIMESEC").getTime());
-    Assert.assertEquals(0, actualResult.getTimestamp("TIMENANO").getNanos());
-    Assert.assertEquals(0, actualResult.getTimestamp("TIMENANO").getTime());
+  private static String nextString(Random r) {
+    return new String(nextBytes(r));
   }
 
-  private void assertNullRow(Map<String, Object> expectedNullRow, ResultSet actualResult)
-      throws SQLException {
-    Assert.assertNotNull(actualResult);
-    assertNonTimeAndVarFields(expectedNullRow, actualResult);
-    Assert.assertEquals(null, actualResult.getString("VAR"));
-    Assert.assertEquals(null, actualResult.getString("OBJ"));
-    Assert.assertEquals(null, actualResult.getString("ARR"));
-    Assert.assertEquals(null, actualResult.getDate("EPOCHDAYS"));
-    Assert.assertEquals(null, actualResult.getTimestamp("EPOCHSEC"));
-    Assert.assertEquals(null, actualResult.getTimestamp("EPOCHNANO"));
-    Assert.assertEquals(null, actualResult.getTimestamp("TIMESEC"));
-    Assert.assertEquals(null, actualResult.getTimestamp("TIMENANO"));
+  private static byte[] nextBytes(Random r) {
+    byte[] bin = new byte[128];
+    r.nextBytes(bin);
+    for (int i = 0; i < bin.length; i++) {
+      bin[i] = (byte) (Math.abs(bin[i]) % 25 + 97); // ascii letters
+    }
+    return bin;
   }
 
-  private void assertPosRow(Map<String, Object> expectedPosRow, ResultSet actualResult)
-      throws SQLException {
-    String formattedJSON =
-        "{\n"
-            + "  \"a\": 1,\n"
-            + "  \"b\": \"qwerty\",\n"
-            + "  \"c\": null,\n"
-            + "  \"d\": {\n"
-            + "    \"e\": 2,\n"
-            + "    \"f\": \"asdf\",\n"
-            + "    \"g\": null\n"
-            + "  }\n"
-            + "}";
-
-    String formattedArray =
-        "[\n"
-            + "  \"{ \\\"a\\\": 1}\",\n"
-            + "  \"{ \\\"b\\\": 2 }\",\n"
-            + "  \"{ \\\"c\\\": 3 }\"\n"
-            + "]";
-    Assert.assertNotNull(actualResult);
-    assertNonTimeAndVarFields(expectedPosRow, actualResult);
-    Assert.assertEquals(formattedJSON, actualResult.getString("VAR"));
-    Assert.assertEquals(formattedJSON, actualResult.getString("OBJ"));
-    Assert.assertEquals(formattedArray, actualResult.getString("ARR"));
-    Assert.assertEquals(
-        1663459200000l, actualResult.getDate("EPOCHDAYS").getTime()); // in ms, 18.09.2022 00:00:00
-    Assert.assertEquals(1663531507000L, actualResult.getTimestamp("EPOCHSEC").getTime());
-    Assert.assertEquals(999999999L, actualResult.getTimestamp("EPOCHNANO").getNanos());
-    // 1663531507000 ms + 999 ms (from ns)
-    Assert.assertEquals(1663531507999L, actualResult.getTimestamp("EPOCHNANO").getTime());
-    Assert.assertEquals(3601000, actualResult.getTimestamp("TIMESEC").getTime()); // 1h + 1s in ms
-    Assert.assertEquals(999999999, actualResult.getTimestamp("TIMENANO").getNanos());
-    // 1h + 1s + 999 ms (from ns)
-    Assert.assertEquals(3601999, actualResult.getTimestamp("TIMENANO").getTime());
+  private static double nextFloat(Random r) {
+    return (r.nextLong() % Math.round(Math.pow(10, 10))) / 100000d;
   }
 
-  private void assertNonTimeAndVarFields(Map<String, Object> row, ResultSet result)
-      throws SQLException {
-    Assert.assertNotNull(result);
-    Assert.assertEquals(row.get("num"), result.getInt("NUM"));
-    Assert.assertEquals(row.get("num_10_5"), result.getBigDecimal("NUM_10_5"));
-    Assert.assertEquals(row.get("num_38_0"), result.getBigDecimal("NUM_38_0"));
-    Assert.assertEquals(row.get("num_2_0"), result.getInt("NUM_2_0"));
-    Assert.assertEquals(row.get("num_4_0"), result.getInt("NUM_4_0"));
-    Assert.assertEquals(row.get("num_9_0"), result.getInt("NUM_9_0"));
-    Assert.assertEquals((long) row.get("num_18_0"), result.getLong("NUM_18_0"));
-    Assert.assertEquals((float) row.get("num_float"), result.getFloat("NUM_FLOAT"), 0);
-    Assert.assertEquals(row.get("str_varchar"), result.getString("STR_VARCHAR"));
-    Assert.assertArrayEquals((byte[]) row.get("bin"), result.getBytes("BIN"));
-    Assert.assertEquals(row.get("bl"), result.getBoolean("BL"));
+  private static String nextJson(Random r) {
+    return String.format(
+        "{ \"%s\": %d, \"%s\": \"%s\", \"%s\": null, \"%s\": { \"%s\": %f, \"%s\": \"%s\", \"%s\":"
+            + " null } }",
+        nextString(r),
+        r.nextInt(),
+        nextString(r),
+        nextString(r),
+        nextString(r),
+        nextString(r),
+        nextString(r),
+        r.nextFloat(),
+        nextString(r),
+        nextString(r),
+        nextString(r));
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index 31afda1f0..0ea0e5de4 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -9,8 +9,6 @@
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.Arrays;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
@@ -21,27 +19,12 @@
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
-import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.SFException;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 
-@RunWith(Parameterized.class)
 public abstract class AbstractDataTypeTest {
-  @Parameterized.Parameters(name = "{0}")
-  public static Collection<Object[]> bdecVersion() {
-    return Arrays.asList(
-        new Object[][] {
-          {"Arrow", Constants.BdecVersion.ONE},
-          // TODO: uncomment once SNOW-659721 is deployed and we set the parameter
-          // DISABLE_PARQUET_CACHE to true for the test account
-          // {"Parquet", Constants.BdecVersion.THREE}
-        });
-  }
-
   private static final String SOURCE_COLUMN_NAME = "source";
   private static final String VALUE_COLUMN_NAME = "value";
 
@@ -69,13 +52,6 @@ protected String randomString() {
 
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
-  private final Constants.BdecVersion bdecVersion;
-
-  public AbstractDataTypeTest(
-      @SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
-    this.bdecVersion = bdecVersion;
-  }
-
   @Before
   public void before() throws Exception {
     databaseName = String.format("SDK_DATATYPE_COMPATIBILITY_IT_%s", randomString());
@@ -89,13 +65,7 @@ public void before() throws Exception {
 
     conn.createStatement().execute(String.format("use warehouse %s;", TestUtils.getWarehouse()));
 
-    if (bdecVersion == Constants.BdecVersion.THREE) {
-      // TODO: encryption and interleaved mode are not yet supported by server side's Parquet
-      // scanner if local file cache is enabled (SNOW-656500)
-      conn.createStatement().execute("alter session set disable_parquet_cache=true;");
-    }
-
-    Properties props = TestUtils.getProperties(bdecVersion);
+    Properties props = TestUtils.getProperties();
     if (props.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       props.setProperty(ROLE, "ACCOUNTADMIN");
     }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
index a7b9d6cd7..f0b0d8199 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
@@ -1,15 +1,10 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Ignore;
 import org.junit.Test;
 
 public class BinaryIT extends AbstractDataTypeTest {
 
-  public BinaryIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testBinarySimple() throws Exception {
     testJdbcTypeCompatibility("BINARY", new byte[0], new ByteArrayProvider());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
index 3a9445ea2..bad5eac16 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
@@ -8,7 +8,6 @@
 import java.time.OffsetTime;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Ignore;
 import org.junit.Test;
 
@@ -18,10 +17,6 @@
  */
 public class DateTimeIT extends AbstractDataTypeTest {
 
-  public DateTimeIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testTimestampWithTimeZone() throws Exception {
     useLosAngelesTimeZone();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
index 48936b950..1fdd2ad5e 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
@@ -2,15 +2,9 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class LogicalTypesIT extends AbstractDataTypeTest {
-
-  public LogicalTypesIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testLogicalTypes() throws Exception {
     // Test booleans
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
index 22c59d3ea..57867427d 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
@@ -1,15 +1,9 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
 import java.util.Arrays;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class NullIT extends AbstractDataTypeTest {
-
-  public NullIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testNullIngestion() throws Exception {
     for (String type :
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
index 64c9bc3fc..1b1c8d280 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
@@ -2,15 +2,9 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class NumericTypesIT extends AbstractDataTypeTest {
-
-  public NumericTypesIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testIntegers() throws Exception {
     // test bytes
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
index 1195d2bfc..3c4ddde22 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
@@ -15,7 +15,6 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -25,10 +24,6 @@ public class SemiStructuredIT extends AbstractDataTypeTest {
   // server-side representation. Validation leaves a small buffer for this difference.
   private static final int MAX_ALLOWED_LENGTH = 16 * 1024 * 1024 - 64;
 
-  public SemiStructuredIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testVariant() throws Exception {
     // Test dates
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
index 198d1c99c..4bcf39519 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
@@ -2,16 +2,10 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Ignore;
 import org.junit.Test;
 
 public class StringsIT extends AbstractDataTypeTest {
-
-  public StringsIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testStrings() throws Exception {
     testJdbcTypeCompatibility("VARCHAR", "", new StringProvider());

From 69c7265bdd79f20ab3b4968b692ca286913e905b Mon Sep 17 00:00:00 2001
From: Snyk bot <snyk-bot@snyk.io>
Date: Thu, 27 Oct 2022 23:22:57 +0300
Subject: [PATCH 082/356] [Snyk] Security upgrade
 org.apache.arrow:arrow-compression from 9.0.0 to 10.0.0 (#271)

fix: pom.xml to reduce vulnerabilities

The following vulnerabilities are fixed with an upgrade:
- https://snyk.io/vuln/SNYK-JAVA-ORGAPACHECOMMONS-1316638
- https://snyk.io/vuln/SNYK-JAVA-ORGAPACHECOMMONS-1316639
- https://snyk.io/vuln/SNYK-JAVA-ORGAPACHECOMMONS-1316640
- https://snyk.io/vuln/SNYK-JAVA-ORGAPACHECOMMONS-1316641
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index cb6b2eda0..265970691 100644
--- a/pom.xml
+++ b/pom.xml
@@ -42,7 +42,7 @@
         <shadeBase>net.snowflake.ingest.internal</shadeBase>
         <jacoco.skip.instrument>true</jacoco.skip.instrument>
         <jacoco.version>0.8.5</jacoco.version>
-        <arrow.version>9.0.0</arrow.version>
+        <arrow.version>10.0.0</arrow.version>
     </properties>
 
     <build>

From 70f32d944f202328a88d8bd7092abc2fcc40c3ff Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Mon, 7 Nov 2022 11:53:44 +0100
Subject: [PATCH 083/356] SNOW-637237 Thread-safe HTTP client initialisation
 (#277)

---
 .../net/snowflake/ingest/utils/HttpUtil.java  | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index f96a32489..a8d08849f 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -7,6 +7,8 @@
 import static net.snowflake.ingest.utils.Utils.isNullOrEmpty;
 
 import java.security.Security;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
 import java.util.Properties;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
@@ -49,8 +51,9 @@ public class HttpUtil {
   public static final String HTTP_PROXY_PASSWORD = "http.proxyPassword";
 
   private static final String PROXY_SCHEME = "http";
+  private static final Duration RETRY_INTERVAL = Duration.of(3, ChronoUnit.SECONDS);
   private static final int MAX_RETRIES = 3;
-  private static CloseableHttpClient httpClient;
+  private static volatile CloseableHttpClient httpClient;
 
   private static PoolingHttpClientConnectionManager connectionManager;
 
@@ -80,7 +83,11 @@ public class HttpUtil {
 
   public static CloseableHttpClient getHttpClient() {
     if (httpClient == null) {
-      initHttpClient();
+      synchronized (HttpUtil.class) {
+        if (httpClient == null) {
+          initHttpClient();
+        }
+      }
     }
 
     initIdleConnectionMonitoringThread();
@@ -189,7 +196,6 @@ private static void initIdleConnectionMonitoringThread() {
 
   private static ServiceUnavailableRetryStrategy getServiceUnavailableRetryStrategy() {
     return new ServiceUnavailableRetryStrategy() {
-      private int executionCount = 0;
       final int REQUEST_TIMEOUT = 408;
       final int TOO_MANY_REQUESTS = 429;
       final int SERVER_ERRORS = 500;
@@ -197,7 +203,6 @@ private static ServiceUnavailableRetryStrategy getServiceUnavailableRetryStrateg
       @Override
       public boolean retryRequest(
           final HttpResponse response, final int executionCount, final HttpContext context) {
-        this.executionCount = executionCount;
         int statusCode = response.getStatusLine().getStatusCode();
         if (executionCount == MAX_RETRIES + 1) {
           LOGGER.info("Reached the max retry time, not retrying anymore");
@@ -209,7 +214,7 @@ public boolean retryRequest(
                     || statusCode >= SERVER_ERRORS)
                 && executionCount < MAX_RETRIES + 1;
         if (needNextRetry) {
-          long interval = (1 << executionCount) * 1000;
+          long interval = getRetryInterval();
           LOGGER.warn(
               "In retryRequest for service unavailability with statusCode:{} and uri:{}",
               statusCode,
@@ -220,11 +225,8 @@ public boolean retryRequest(
       }
 
       @Override
-      // The waiting time is backoff, and is
-      // the exponential of the executionCount.
       public long getRetryInterval() {
-        long interval = (1 << executionCount) * 1000; // milliseconds
-        return interval;
+        return RETRY_INTERVAL.toMillis();
       }
     };
   }

From 9f1dfa98ebe87f850c369de0674a0933523f6705 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Tue, 8 Nov 2022 01:11:47 -0800
Subject: [PATCH 084/356] SNOW-684474: Support adding a memory limit through
 the config file (#279)

- Support adding a memory limit per client through the config file
- Comparing the free memory with the max memory now instead of total memory for the throttling logic, assuming that the memory could grow to max memory
---
 .../streaming/internal/AbstractRowBuffer.java  |  2 +-
 ...nowflakeStreamingIngestChannelInternal.java | 18 ++++++++++++++++--
 .../net/snowflake/ingest/utils/Cryptor.java    |  2 +-
 .../ingest/utils/ParameterProvider.java        | 13 +++++++++++++
 .../internal/ParameterProviderTest.java        | 11 ++++++++---
 .../SnowflakeStreamingIngestChannelTest.java   | 13 +++++++++++--
 6 files changed, 50 insertions(+), 9 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 064967e43..2800310e5 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -271,7 +271,7 @@ public InsertValidationResponse insertRows(
             error.setException(e);
             response.addError(error);
           } catch (Throwable e) {
-            logger.logWarn("Unexpected error happens during insertRows: {}", e.getMessage());
+            logger.logWarn("Unexpected error happens during insertRows: {}", e);
             error.setException(new SFException(e, ErrorCode.INTERNAL_ERROR, e.getMessage()));
             response.addError(error);
           }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index d2d64db29..82566de02 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -8,6 +8,7 @@
 import static net.snowflake.ingest.utils.Constants.LOW_RUNTIME_MEMORY_THRESHOLD_IN_BYTES;
 import static net.snowflake.ingest.utils.Constants.MAX_CHUNK_SIZE_IN_BYTES;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
+import static net.snowflake.ingest.utils.ParameterProvider.MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT;
 
 import java.util.Collections;
 import java.util.List;
@@ -445,16 +446,29 @@ void throttleInsertIfNeeded(Runtime runtime) {
         throw new SFException(ErrorCode.INTERNAL_ERROR, "Insert throttle get interrupted");
       }
     }
+    if (retry > 0) {
+      logger.logInfo(
+          "Insert throttled for a total of {} milliseconds, retryCount={}, client={}, channel={}",
+          retry * insertThrottleIntervalInMs,
+          retry,
+          this.owningClient.getName(),
+          getFullyQualifiedName());
+    }
   }
 
   /** Check whether we have a low runtime memory condition */
   private boolean hasLowRuntimeMemory(Runtime runtime) {
     int insertThrottleThresholdInPercentage =
         this.owningClient.getParameterProvider().getInsertThrottleThresholdInPercentage();
+    long maxMemoryLimitInBytes =
+        this.owningClient.getParameterProvider().getMaxMemoryLimitInBytes();
+    long maxMemory =
+        maxMemoryLimitInBytes == MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT
+            ? runtime.maxMemory()
+            : maxMemoryLimitInBytes;
     boolean hasLowRuntimeMemory =
         runtime.freeMemory() < LOW_RUNTIME_MEMORY_THRESHOLD_IN_BYTES
-            && runtime.freeMemory() * 100 / runtime.totalMemory()
-                < insertThrottleThresholdInPercentage;
+            && runtime.freeMemory() * 100 / maxMemory < insertThrottleThresholdInPercentage;
     if (hasLowRuntimeMemory) {
       logger.logWarn(
           "Throttled due to memory pressure, client={}, channel={}.",
diff --git a/src/main/java/net/snowflake/ingest/utils/Cryptor.java b/src/main/java/net/snowflake/ingest/utils/Cryptor.java
index 92c4429ab..74a033fe1 100644
--- a/src/main/java/net/snowflake/ingest/utils/Cryptor.java
+++ b/src/main/java/net/snowflake/ingest/utils/Cryptor.java
@@ -103,7 +103,7 @@ public static byte[] encrypt(
     // Generate the derived key
     SecretKey derivedKey = deriveKey(encryptionKey, diversifier);
 
-    // Encrypt with zero IV
+    // Encrypt with the corresponding IV
     Cipher cipher = Cipher.getInstance(ENCRYPTION_ALGORITHM);
     byte[] ivBytes = ByteBuffer.allocate(2 * Long.BYTES).putLong(0).putLong(iv).array();
     cipher.init(Cipher.ENCRYPT_MODE, derivedKey, new IvParameterSpec(ivBytes));
diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
index 1ffc25eab..85cb22575 100644
--- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
+++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -20,6 +20,7 @@ public class ParameterProvider {
   public static final String IO_TIME_CPU_RATIO = "IO_TIME_CPU_RATIO".toLowerCase();
   public static final String BLOB_UPLOAD_MAX_RETRY_COUNT =
       "BLOB_UPLOAD_MAX_RETRY_COUNT".toLowerCase();
+  public static final String MAX_MEMORY_LIMIT_IN_BYTES = "MAX_MEMORY_LIMIT_IN_BYTES".toLowerCase();
 
   // Default values
   public static final long BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT = 1000;
@@ -30,6 +31,7 @@ public class ParameterProvider {
   public static final Constants.BdecVersion BLOB_FORMAT_VERSION_DEFAULT = Constants.BdecVersion.ONE;
   public static final int IO_TIME_CPU_RATIO_DEFAULT = 2;
   public static final int BLOB_UPLOAD_MAX_RETRY_COUNT_DEFAULT = 24;
+  public static final long MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT = -1L;
 
   /** Map of parameter name to parameter value. This will be set by client/configure API Call. */
   private final Map<String, Object> parameterMap = new HashMap<>();
@@ -108,6 +110,9 @@ private void setParameterMap(Map<String, Object> parameterOverrides, Properties
         BLOB_UPLOAD_MAX_RETRY_COUNT_DEFAULT,
         parameterOverrides,
         props);
+
+    this.updateValue(
+        MAX_MEMORY_LIMIT_IN_BYTES, MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT, parameterOverrides, props);
   }
 
   /** @return Longest interval in milliseconds between buffer flushes */
@@ -206,6 +211,14 @@ public int getBlobUploadMaxRetryCount() {
     return (int) val;
   }
 
+  /** @return The max memory limit in bytes */
+  public long getMaxMemoryLimitInBytes() {
+    Object val =
+        this.parameterMap.getOrDefault(
+            MAX_MEMORY_LIMIT_IN_BYTES, MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT);
+    return (val instanceof String) ? Long.parseLong(val.toString()) : (long) val;
+  }
+
   @Override
   public String toString() {
     return "ParameterProvider{" + "parameterMap=" + parameterMap + '}';
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
index 7a506a1d1..b7b1a1c74 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
@@ -19,14 +19,16 @@ public void withValuesSet() {
     parameterMap.put(ParameterProvider.INSERT_THROTTLE_INTERVAL_IN_MILLIS, 7L);
     parameterMap.put(ParameterProvider.IO_TIME_CPU_RATIO, 10);
     parameterMap.put(ParameterProvider.BLOB_UPLOAD_MAX_RETRY_COUNT, 100);
+    parameterMap.put(ParameterProvider.MAX_MEMORY_LIMIT_IN_BYTES, 1000L);
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
 
-    Assert.assertEquals(3, parameterProvider.getBufferFlushIntervalInMs());
-    Assert.assertEquals(4, parameterProvider.getBufferFlushCheckIntervalInMs());
+    Assert.assertEquals(3L, parameterProvider.getBufferFlushIntervalInMs());
+    Assert.assertEquals(4L, parameterProvider.getBufferFlushCheckIntervalInMs());
     Assert.assertEquals(6, parameterProvider.getInsertThrottleThresholdInPercentage());
-    Assert.assertEquals(7, parameterProvider.getInsertThrottleIntervalInMs());
+    Assert.assertEquals(7L, parameterProvider.getInsertThrottleIntervalInMs());
     Assert.assertEquals(10, parameterProvider.getIOTimeCpuRatio());
     Assert.assertEquals(100, parameterProvider.getBlobUploadMaxRetryCount());
+    Assert.assertEquals(1000L, parameterProvider.getMaxMemoryLimitInBytes());
   }
 
   @Test
@@ -100,5 +102,8 @@ public void withDefaultValues() {
     Assert.assertEquals(
         ParameterProvider.BLOB_UPLOAD_MAX_RETRY_COUNT_DEFAULT,
         parameterProvider.getBlobUploadMaxRetryCount());
+    Assert.assertEquals(
+        ParameterProvider.MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT,
+        parameterProvider.getMaxMemoryLimitInBytes());
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 5c341e4c9..3a86ef57a 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -530,7 +530,7 @@ public void testInsertRowThrottling() {
             OpenChannelRequest.OnErrorOption.CONTINUE);
 
     Runtime mockedRunTime = Mockito.mock(Runtime.class);
-    Mockito.when(mockedRunTime.totalMemory()).thenReturn(1000000L);
+    Mockito.when(mockedRunTime.maxMemory()).thenReturn(1000000L);
     ParameterProvider parameterProvider = new ParameterProvider();
     Mockito.when(mockedRunTime.freeMemory())
         .thenReturn(
@@ -540,12 +540,21 @@ public void testInsertRowThrottling() {
         CompletableFuture.runAsync(() -> channel.throttleInsertIfNeeded(mockedRunTime));
 
     try {
-      future.get(1L, TimeUnit.SECONDS);
+      future.get(5L, TimeUnit.SECONDS);
       Assert.fail("the insert should be throttled.");
     } catch (TimeoutException ignored) {
     } catch (Exception e) {
       Assert.fail("unexpected exception encountered.");
     }
+
+    Mockito.when(mockedRunTime.freeMemory()).thenReturn(1000000L);
+
+    // We should succeed now
+    try {
+      future.get(5L, TimeUnit.SECONDS);
+    } catch (Exception e) {
+      Assert.fail("unexpected exception encountered.");
+    }
   }
 
   @Test

From df0838c22e94a2ceee5d1443972bf5292f8ab612 Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Tue, 25 Oct 2022 13:08:02 +0000
Subject: [PATCH 085/356] @snow SNOW-633432 Parquet file generator (fixed dep)

---
 pom.xml                                       |  86 +++
 scripts/check_content.sh                      |  24 +-
 .../streaming/internal/ArrowRowBuffer.java    |  10 +-
 .../streaming/internal/BlobMetadata.java      |  17 +-
 .../internal/BlobMetadataWithBdecVersion.java |  25 +
 .../streaming/internal/ChannelCache.java      |   3 +-
 .../internal/DataValidationUtil.java          |  11 +
 .../streaming/internal/FlushService.java      |   7 +-
 .../ingest/streaming/internal/Flusher.java    |   3 +-
 .../streaming/internal/ParquetChunkData.java  |  25 +
 .../streaming/internal/ParquetFlusher.java    | 348 +++++++++++
 .../streaming/internal/ParquetRowBuffer.java  | 223 +++++++
 .../internal/ParquetTypeGenerator.java        | 292 +++++++++
 .../internal/ParquetValueParser.java          | 323 ++++++++++
 .../streaming/internal/RegisterService.java   |   3 +-
 .../ingest/streaming/internal/RowBuffer.java  |   3 +-
 ...owflakeStreamingIngestChannelInternal.java |  20 +-
 ...nowflakeStreamingIngestClientInternal.java |   5 +-
 .../net/snowflake/ingest/utils/Constants.java |   9 +-
 .../java/net/snowflake/ingest/TestUtils.java  |  12 +-
 .../streaming/internal/ArrowBufferTest.java   | 242 ++++----
 .../internal/ColumnMetadataBuilder.java       | 164 +++++
 .../streaming/internal/FlushServiceTest.java  |  50 +-
 .../internal/ParquetTypeGeneratorTest.java    | 480 +++++++++++++++
 .../internal/ParquetValueParserTest.java      | 570 ++++++++++++++++++
 .../streaming/internal/RowBufferTest.java     |   7 +-
 .../streaming/internal/StreamingIngestIT.java | 289 ++++++---
 .../datatypes/AbstractDataTypeTest.java       |  32 +-
 .../internal/datatypes/BinaryIT.java          |   5 +
 .../internal/datatypes/DateTimeIT.java        |   5 +
 .../internal/datatypes/LogicalTypesIT.java    |   6 +
 .../streaming/internal/datatypes/NullIT.java  |   6 +
 .../internal/datatypes/NumericTypesIT.java    |   6 +
 .../internal/datatypes/SemiStructuredIT.java  |   5 +
 .../internal/datatypes/StringsIT.java         |   6 +
 35 files changed, 3060 insertions(+), 262 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java

diff --git a/pom.xml b/pom.xml
index 265970691..ee0cd44f6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -43,6 +43,7 @@
         <jacoco.skip.instrument>true</jacoco.skip.instrument>
         <jacoco.version>0.8.5</jacoco.version>
         <arrow.version>10.0.0</arrow.version>
+        <hadoop.version>3.3.4</hadoop.version>
     </properties>
 
     <build>
@@ -292,6 +293,91 @@
             <version>${arrow.version}</version>
         </dependency>
 
+        <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop -->
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-hadoop</artifactId>
+            <version>1.12.3</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>javax.annotation</groupId>
+                    <artifactId>javax.annotation-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-mapreduce-client-core</artifactId>
+            <version>${hadoop.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-yarn-common</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-hdfs-client</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>${hadoop.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.github.spotbugs</groupId>
+                    <artifactId>spotbugs-annotations</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-json</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-server</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>javax.servlet</groupId>
+                    <artifactId>servlet-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>javax.servlet.jsp</groupId>
+                    <artifactId>jsp-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.codehaus.jackson</groupId>
+                    <artifactId>jackson-mapper-asl</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.mortbay.jetty</groupId>
+                    <artifactId>jetty</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.zookeeper</groupId>
+                    <artifactId>zookeeper</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
         <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-core -->
         <dependency>
             <groupId>io.dropwizard.metrics</groupId>
diff --git a/scripts/check_content.sh b/scripts/check_content.sh
index 9e5b1c404..09022d9d8 100755
--- a/scripts/check_content.sh
+++ b/scripts/check_content.sh
@@ -5,7 +5,29 @@
 set -o pipefail
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-if jar tvf $DIR/../target/snowflake-ingest-sdk.jar  | awk '{print $8}' | grep -v -E "^(net|com)/snowflake" | grep -v -E "(com|net)/\$" | grep -v -E "^META-INF" | grep -v -E "^mozilla" | grep -v mime.types | grep -v project.properties | grep -v -E "javax" | grep -v -E "^org/" | grep -v -E "^com/google" | grep -v -E "^com/sun" | grep -v "log4j.properties" | grep -v "git.properties" | grep -v "io/" | grep -v "codegen/" | grep -v "com/codahale/" | grep -v "com/ibm/" | grep -v "LICENSE" | grep -v "aix/" | grep -v "darwin/" | grep -v "win/" | grep -v "freebsd/" | grep -v "linux/" | grep -v "com/github/"; then
+if jar tvf $DIR/../target/snowflake-ingest-sdk.jar  | awk '{print $8}' | grep -v -E "^(net|com)/snowflake" \
+    | grep -v -E "(com|net)/\$" | grep -v -E "^META-INF" | grep -v -E "^mozilla" | grep -v mime.types \
+    | grep -v project.properties | grep -v -E "javax" | grep -v -E "^org/" | grep -v -E "^com/google" \
+    | grep -v -E "^com/sun" | grep -v "log4j.properties" | grep -v "git.properties" | grep -v "io/" \
+    | grep -v "codegen/" | grep -v "com/codahale/" | grep -v "com/ibm/" | grep -v "LICENSE" | grep -v "aix/" \
+    | grep -v "darwin/" | grep -v "win/" | grep -v "freebsd/" | grep -v "linux/" | grep -v "com/github/" \
+    | grep -v -E "shaded/" | grep -v "webapps/"  | grep -v "microsoft/" | grep -v "com/ctc/" | grep -v "jersey/" \
+    | grep -v "edu/" | grep -v "com/jcraft/" | grep -v "contribs/" | grep -v "com/zaxxer/"  | grep -v "com/squareup/" \
+    | grep -v "com/thoughtworks/" | grep -v "com/jamesmurty/"  | grep -v "net/iharder/" \
+    | grep -v -E "^core-default.xml" \
+    | grep -v "yarn-version-info.properties" | grep -v "yarn-version-info.properties" \
+    | grep -v "about.html" | grep -v "jetty-dir.css" \
+    | grep -v "krb5_udp-template.conf" | grep -v "krb5-template.conf" \
+    | grep -v "ccache.txt" | grep -v "keytab.txt" \
+    | grep -v "common-version-info.properties" | grep -v "LocalizedFormats_fr.properties" \
+    | grep -v "org.apache.hadoop.application-classloader.properties" | grep -v "assets/" | grep -v "ehcache-core.xsd" \
+    | grep -v "ehcache-107ext.xsd" | grep -v "parquet.thrift" | grep -v "mapred-default.xml" \
+    | grep -v "yarn-default.xml" | grep -v ".keep"  | grep -v "NOTICE" | grep -v "digesterRules.xml" \
+    | grep -v "properties.dtd" | grep -v "PropertyList-1.0.dtd"  \
+    | grep -v "about.html" | grep -v "jetty-dir.css" | grep -v "krb5-template.conf" \
+    | grep -v "krb5_udp-template.conf" | grep -v "ccache.txt" | grep -v "keytab.txt" \
+    | grep -v "lookup.class" | grep -v "update.class" | grep -v "dig.class" | grep -v "jnamed"  \
+    ; then
   echo "[ERROR] Ingest SDK jar includes class not under the snowflake namespace"
   exit 1
 fi
\ No newline at end of file
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 38b4583a9..2d7d17a92 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -471,15 +471,7 @@ private float convertRowToArrow(
             // vector.setSafe requires the BigDecimal input scale explicitly match its scale
             inputAsBigDecimal = inputAsBigDecimal.setScale(columnScale, RoundingMode.HALF_UP);
 
-            if (inputAsBigDecimal.abs().compareTo(BigDecimal.TEN.pow(columnPrecision - columnScale))
-                >= 0) {
-              throw new SFException(
-                  ErrorCode.INVALID_ROW,
-                  inputAsBigDecimal,
-                  String.format(
-                      "Number out of representable exclusive range of (-1e%s..1e%s)",
-                      columnPrecision - columnScale, columnPrecision - columnScale));
-            }
+            DataValidationUtil.checkValueInRange(inputAsBigDecimal, columnScale, columnPrecision);
 
             if (columnScale != 0 || physicalType == ColumnPhysicalType.SB16) {
               ((DecimalVector) vector).setSafe(curRowIndex, inputAsBigDecimal);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index 2818a392a..f90ba5055 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -49,9 +49,16 @@ List<ChunkMetadata> getChunks() {
     return this.chunks;
   }
 
-  // TODO: send the bdec_version once server side supports this in production
-  //  @JsonProperty("bdec_version")
-  //  byte getVersionByte() {
-  //    return bdecVersion.toByte();
-  //  }
+  /**
+   * Create {@link BlobMetadata} in case of {@link Constants.BdecVersion#ONE} and {@link
+   * BlobMetadataWithBdecVersion} otherwise to send BDEC version to server side.
+   */
+  static BlobMetadata createBlobMetadata(
+      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks) {
+    // TODO SNOW-659721: Unify BlobMetadata with BlobMetadataWithBdecVersion once server side
+    // BdecVersion in production
+    return bdecVersion == Constants.BdecVersion.ONE
+        ? new BlobMetadata(path, md5, bdecVersion, chunks)
+        : new BlobMetadataWithBdecVersion(path, md5, bdecVersion, chunks);
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
new file mode 100644
index 000000000..7f9c602e3
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+import net.snowflake.ingest.utils.Constants;
+
+/**
+ * Same as {@link BlobMetadata} but with {@link Constants.BdecVersion} to send it to server side if
+ * the version is greater than {@link Constants.BdecVersion#ONE}.
+ */
+class BlobMetadataWithBdecVersion extends BlobMetadata {
+  BlobMetadataWithBdecVersion(
+      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks) {
+    super(path, md5, bdecVersion, chunks);
+  }
+
+  @JsonProperty("bdec_version")
+  byte getVersionByte() {
+    return getVersion().toByte();
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
index 35ca580bc..ef366b120 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
@@ -14,7 +14,8 @@
  * during flush. The key is a fully qualified table name and the value is a set of channels that
  * belongs to this table
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 class ChannelCache<T> {
   // Cache to hold all the valid channels, the key for the outer map is FullyQualifiedTableName and
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 881a4c2f3..de7185009 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -674,6 +674,17 @@ static int validateAndParseBoolean(Object input) {
         input.getClass(), "BOOLEAN", new String[] {"boolean", "Number", "String"});
   }
 
+  static void checkValueInRange(BigDecimal bigDecimalValue, int scale, int precision) {
+    if (bigDecimalValue.abs().compareTo(BigDecimal.TEN.pow(precision - scale)) >= 0) {
+      throw new SFException(
+          ErrorCode.INVALID_ROW,
+          bigDecimalValue,
+          String.format(
+              "Number out of representable exclusive range of (-1e%s..1e%s)",
+              precision - scale, precision - scale));
+    }
+  }
+
   static Set<String> allowedBooleanStringsLowerCased =
       Sets.newHashSet("1", "0", "yes", "no", "y", "n", "t", "f", "true", "false", "on", "off");
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index df5522469..0ef790c6f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -58,7 +58,8 @@
  * <li>upload the blob to stage
  * <li>register the blob to the targeted Snowflake table
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 class FlushService<T> {
 
@@ -467,7 +468,6 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
         // We need to maintain IV as a block counter for the whole file, even interleaved,
         // to align with decryption on the Snowflake query path.
         // TODO: address alignment for the header SNOW-557866
-        // TODO: encryption is not yet supported by server side for Parquet
         long iv = curDataSize / Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES;
         byte[] encryptedCompressedChunkData =
             Cryptor.encrypt(
@@ -559,7 +559,8 @@ BlobMetadata upload(String filePath, byte[] blob, List<ChunkMetadata> metadata)
         blob.length,
         System.currentTimeMillis() - startTime);
 
-    return new BlobMetadata(filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata);
+    return BlobMetadata.createBlobMetadata(
+        filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
index 4ac7acf8c..37ab4afc4 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
@@ -13,7 +13,8 @@
  * Interface to convert {@link ChannelData} buffered in {@link RowBuffer} to the underlying format
  * implementation for faster processing.
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 public interface Flusher<T> {
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
new file mode 100644
index 000000000..0c7b12ff2
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.util.List;
+import java.util.Map;
+
+/** Parquet data holder to buffer rows. */
+public class ParquetChunkData {
+  final List<List<Object>> rows;
+  final Map<String, String> metadata;
+
+  /**
+   * Construct parquet data chunk.
+   *
+   * @param rows chunk row set
+   * @param metadata chunk metadata
+   */
+  public ParquetChunkData(List<List<Object>> rows, Map<String, String> metadata) {
+    this.rows = rows;
+    this.metadata = metadata;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
new file mode 100644
index 000000000..3d9827b6b
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
@@ -0,0 +1,348 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import static net.snowflake.ingest.utils.Constants.MAX_CHUNK_SIZE_IN_BYTES;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.ParquetProperties;
+import org.apache.parquet.hadoop.ParquetFileWriter;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.api.WriteSupport;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.io.DelegatingPositionOutputStream;
+import org.apache.parquet.io.OutputFile;
+import org.apache.parquet.io.ParquetEncodingException;
+import org.apache.parquet.io.PositionOutputStream;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.api.RecordConsumer;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+
+/**
+ * Converts {@link ChannelData} buffered in {@link RowBuffer} to the Parquet format for faster
+ * processing.
+ */
+public class ParquetFlusher implements Flusher<ParquetChunkData> {
+  private static final Logging logger = new Logging(ParquetFlusher.class);
+  private final MessageType schema;
+
+  /** Construct parquet flusher from its schema. */
+  public ParquetFlusher(MessageType schema) {
+    this.schema = schema;
+  }
+
+  @Override
+  public SerializationResult serialize(
+      List<ChannelData<ParquetChunkData>> channelsDataPerTable,
+      ByteArrayOutputStream chunkData,
+      String filePath)
+      throws IOException {
+    List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
+    long rowCount = 0L;
+    SnowflakeStreamingIngestChannelInternal<ParquetChunkData> firstChannel = null;
+    Map<String, RowBufferStats> columnEpStatsMapCombined = null;
+    List<List<Object>> rows = null;
+
+    for (ChannelData<ParquetChunkData> data : channelsDataPerTable) {
+      // Create channel metadata
+      ChannelMetadata channelMetadata =
+          ChannelMetadata.builder()
+              .setOwningChannel(data.getChannel())
+              .setRowSequencer(data.getRowSequencer())
+              .setOffsetToken(data.getOffsetToken())
+              .build();
+      // Add channel metadata to the metadata list
+      channelsMetadataList.add(channelMetadata);
+
+      logger.logDebug(
+          "Parquet Flusher: Start building channel={}, rowCount={}, bufferSize={} in blob={}",
+          data.getChannel().getFullyQualifiedName(),
+          data.getRowCount(),
+          data.getBufferSize(),
+          filePath);
+
+      if (rows == null) {
+        columnEpStatsMapCombined = data.getColumnEps();
+        rows = new ArrayList<>();
+        firstChannel = data.getChannel();
+      } else {
+        // This method assumes that channelsDataPerTable is grouped by table. We double check
+        // here and throw an error if the assumption is violated
+        if (!data.getChannel()
+            .getFullyQualifiedTableName()
+            .equals(firstChannel.getFullyQualifiedTableName())) {
+          throw new SFException(ErrorCode.INVALID_DATA_IN_CHUNK);
+        }
+
+        columnEpStatsMapCombined =
+            ChannelData.getCombinedColumnStatsMap(columnEpStatsMapCombined, data.getColumnEps());
+      }
+      rows.addAll(data.getVectors().rows);
+
+      rowCount += data.getRowCount();
+
+      logger.logDebug(
+          "Parquet Flusher: Finish building channel={}, rowCount={}, bufferSize={} in blob={}",
+          data.getChannel().getFullyQualifiedName(),
+          data.getRowCount(),
+          data.getBufferSize(),
+          filePath);
+    }
+
+    Map<String, String> metadata = channelsDataPerTable.get(0).getVectors().metadata;
+
+    flushToParquetBdecChunk(chunkData, rows, metadata, channelsMetadataList);
+    return new SerializationResult(channelsMetadataList, columnEpStatsMapCombined, rowCount);
+  }
+
+  /**
+   * Flushes a parquet row chunk to the given BDEC output stream.
+   *
+   * @param bdecOutput BDEC output stream
+   * @param chunkRows chunk rows
+   * @param metadata chunk metadata
+   * @param channelsMetadataList metadata of the channels the chunk rows belong to
+   * @throws IOException thrown from Parquet library in case of writing problems
+   */
+  private void flushToParquetBdecChunk(
+      ByteArrayOutputStream bdecOutput,
+      List<List<Object>> chunkRows,
+      Map<String, String> metadata,
+      List<ChannelMetadata> channelsMetadataList)
+      throws IOException {
+    try {
+      ParquetWriter<List<Object>> writer =
+          new BdecParquetWriterBuilder(bdecOutput, schema, metadata, channelsMetadataList)
+              // PARQUET_2_0 uses Encoding.DELTA_BYTE_ARRAY for byte arrays (e.g. SF sb16)
+              // server side does not support it TODO: SNOW-657238
+              .withWriterVersion(ParquetProperties.WriterVersion.PARQUET_1_0)
+
+              // the dictionary encoding (Encoding.*_DICTIONARY) is not supported by server side
+              // scanner yet
+              .withDictionaryEncoding(false)
+
+              // Historically server side scanner supports only the case when the row number in all
+              // pages is the same.
+              // The quick fix is to effectively disable the page size/row limit
+              // to always have one page per chunk until server side is generalised.
+              .withPageSize((int) Constants.MAX_CHUNK_SIZE_IN_BYTES * 2)
+              .withPageRowCountLimit(chunkRows.size() + 1)
+              .enableValidation()
+              .withCompressionCodec(CompressionCodecName.GZIP)
+              .withWriteMode(ParquetFileWriter.Mode.CREATE)
+              .build();
+
+      // We can use lower level column writers and custom ValuesWriterFactory that uses plain byte
+      // array encoding used by PARQUET_1_0 and supported by server side
+      // TODO: SNOW-672143
+      for (List<Object> row : chunkRows) {
+        writer.write(row);
+      }
+      writer.close();
+    } catch (Throwable t) {
+      logger.logError("Parquet Flusher: failed to write", t);
+      throw t;
+    }
+  }
+
+  /**
+   * A parquet specific write builder.
+   *
+   * <p>This class is implemented as parquet library API requires, mostly to provide {@link
+   * BdecWriteSupport} implementation.
+   */
+  private static class BdecParquetWriterBuilder
+      extends ParquetWriter.Builder<List<Object>, BdecParquetWriterBuilder> {
+    private final MessageType schema;
+    private final Map<String, String> extraMetaData;
+    private final List<ChannelMetadata> channelsMetadataList;
+
+    protected BdecParquetWriterBuilder(
+        ByteArrayOutputStream stream,
+        MessageType schema,
+        Map<String, String> extraMetaData,
+        List<ChannelMetadata> channelsMetadataList) {
+      super(new ByteArrayOutputFile(stream));
+      this.schema = schema;
+      this.extraMetaData = extraMetaData;
+      this.channelsMetadataList = channelsMetadataList;
+    }
+
+    @Override
+    protected BdecParquetWriterBuilder self() {
+      return this;
+    }
+
+    @Override
+    protected WriteSupport<List<Object>> getWriteSupport(Configuration conf) {
+      return new BdecWriteSupport(schema, extraMetaData, channelsMetadataList);
+    }
+  }
+
+  /**
+   * A parquet specific file output implementation.
+   *
+   * <p>This class is implemented as parquet library API requires, mostly to create our {@link
+   * ByteArrayDelegatingPositionOutputStream} implementation.
+   */
+  private static class ByteArrayOutputFile implements OutputFile {
+    private final ByteArrayOutputStream stream;
+
+    private ByteArrayOutputFile(ByteArrayOutputStream stream) {
+      this.stream = stream;
+    }
+
+    @Override
+    public PositionOutputStream create(long blockSizeHint) throws IOException {
+      stream.reset();
+      return new ByteArrayDelegatingPositionOutputStream(stream);
+    }
+
+    @Override
+    public PositionOutputStream createOrOverwrite(long blockSizeHint) throws IOException {
+      return create(blockSizeHint);
+    }
+
+    @Override
+    public boolean supportsBlockSize() {
+      return false;
+    }
+
+    @Override
+    public long defaultBlockSize() {
+      return (int) MAX_CHUNK_SIZE_IN_BYTES;
+    }
+  }
+
+  /**
+   * A parquet specific output stream implementation.
+   *
+   * <p>This class is implemented as parquet library API requires, mostly to wrap our BDEC output
+   * {@link ByteArrayOutputStream}.
+   */
+  private static class ByteArrayDelegatingPositionOutputStream
+      extends DelegatingPositionOutputStream {
+    private final ByteArrayOutputStream stream;
+
+    public ByteArrayDelegatingPositionOutputStream(ByteArrayOutputStream stream) {
+      super(stream);
+      this.stream = stream;
+    }
+
+    @Override
+    public long getPos() {
+      return stream.size();
+    }
+  }
+
+  /**
+   * A parquet specific write support implementation.
+   *
+   * <p>This class is implemented as parquet library API requires, mostly to serialize user column
+   * values depending on type into Parquet {@link RecordConsumer} in {@link
+   * BdecWriteSupport#write(List)}.
+   */
+  private static class BdecWriteSupport extends WriteSupport<List<Object>> {
+    MessageType schema;
+    RecordConsumer recordConsumer;
+    Map<String, String> extraMetadata;
+    List<ChannelMetadata> channelsMetadataList;
+
+    // TODO SNOW-672156: support specifying encodings and compression
+    BdecWriteSupport(
+        MessageType schema,
+        Map<String, String> extraMetadata,
+        List<ChannelMetadata> channelsMetadataList) {
+      this.schema = schema;
+      this.extraMetadata = extraMetadata;
+      this.channelsMetadataList = channelsMetadataList;
+    }
+
+    @Override
+    public WriteContext init(Configuration config) {
+      return new WriteContext(schema, extraMetadata);
+    }
+
+    @Override
+    public void prepareForWrite(RecordConsumer recordConsumer) {
+      this.recordConsumer = recordConsumer;
+    }
+
+    @Override
+    public void write(List<Object> values) {
+      List<ColumnDescriptor> cols = schema.getColumns();
+      if (values.size() != cols.size()) {
+        List<String> channelNames =
+            this.channelsMetadataList.stream()
+                .map(ChannelMetadata::getChannelName)
+                .collect(Collectors.toList());
+        throw new ParquetEncodingException(
+            "Invalid input data in channels "
+                + channelNames
+                + ". Expecting "
+                + cols.size()
+                + " columns. Input had "
+                + values.size()
+                + " columns ("
+                + cols
+                + ") : "
+                + values);
+      }
+
+      recordConsumer.startMessage();
+      for (int i = 0; i < cols.size(); ++i) {
+        Object val = values.get(i);
+        // val.length() == 0 indicates a NULL value.
+        if (val != null) {
+          String fieldName = cols.get(i).getPath()[0];
+          recordConsumer.startField(fieldName, i);
+          PrimitiveType.PrimitiveTypeName typeName =
+              cols.get(i).getPrimitiveType().getPrimitiveTypeName();
+          switch (typeName) {
+            case BOOLEAN:
+              recordConsumer.addBoolean((boolean) val);
+              break;
+            case FLOAT:
+              recordConsumer.addFloat((float) val);
+              break;
+            case DOUBLE:
+              recordConsumer.addDouble((double) val);
+              break;
+            case INT32:
+              recordConsumer.addInteger((int) val);
+              break;
+            case INT64:
+              recordConsumer.addLong((long) val);
+              break;
+            case BINARY:
+              recordConsumer.addBinary(Binary.fromString((String) val));
+              break;
+            case FIXED_LEN_BYTE_ARRAY:
+              Binary binary = Binary.fromConstantByteArray((byte[]) val);
+              recordConsumer.addBinary(binary);
+              break;
+            default:
+              throw new ParquetEncodingException(
+                  "Unsupported column type: " + cols.get(i).getPrimitiveType());
+          }
+          recordConsumer.endField(fieldName, i);
+        }
+      }
+      recordConsumer.endMessage();
+    }
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
new file mode 100644
index 000000000..3f24d4128
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -0,0 +1,223 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+
+/**
+ * The buffer in the Streaming Ingest channel that holds the un-flushed rows, these rows will be
+ * converted to Parquet format for faster processing
+ */
+public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
+  private static final Logging logger = new Logging(ParquetRowBuffer.class);
+
+  private static final String PARQUET_MESSAGE_TYPE_NAME = "bdec";
+
+  private final Map<String, Pair<ColumnMetadata, Integer>> fieldIndex;
+  private final Map<String, String> metadata;
+  private final List<List<Object>> data;
+  private final List<List<Object>> tempData;
+
+  private MessageType schema;
+
+  /**
+   * Construct a ParquetRowBuffer object
+   *
+   * @param channel client channel
+   */
+  ParquetRowBuffer(SnowflakeStreamingIngestChannelInternal<ParquetChunkData> channel) {
+    super(channel);
+    fieldIndex = new HashMap<>();
+    metadata = new HashMap<>();
+    data = new ArrayList<>();
+    tempData = new ArrayList<>();
+  }
+
+  @Override
+  public void setupSchema(List<ColumnMetadata> columns) {
+    fieldIndex.clear();
+    metadata.clear();
+    metadata.put("sfVer", "1,1");
+    List<Type> parquetTypes = new ArrayList<>();
+    // Snowflake column id that corresponds to the order in 'columns' received from server
+    // id is required to pack column metadata for the server scanner, e.g. decimal scale and
+    // precision
+    int id = 1;
+    for (ColumnMetadata column : columns) {
+      ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+          ParquetTypeGenerator.generateColumnParquetTypeInfo(column, id);
+      parquetTypes.add(typeInfo.getParquetType());
+      this.metadata.putAll(typeInfo.getMetadata());
+      fieldIndex.put(column.getName(), new Pair<>(column, parquetTypes.size() - 1));
+      if (!column.getNullable()) {
+        addNonNullableFieldName(column.getName());
+      }
+      this.statsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
+
+      if (this.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.ABORT) {
+        this.tempStatsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
+      }
+
+      id++;
+    }
+    schema = new MessageType(PARQUET_MESSAGE_TYPE_NAME, parquetTypes);
+  }
+
+  @Override
+  boolean hasColumn(String name) {
+    return fieldIndex.containsKey(name);
+  }
+
+  @Override
+  float addRow(
+      Map<String, Object> row,
+      int curRowIndex,
+      Map<String, RowBufferStats> statsMap,
+      Set<String> formattedInputColumnNames) {
+    return addRow(row, data, statsMap, formattedInputColumnNames);
+  }
+
+  @Override
+  float addTempRow(
+      Map<String, Object> row,
+      int curRowIndex,
+      Map<String, RowBufferStats> statsMap,
+      Set<String> formattedInputColumnNames) {
+    return addRow(row, tempData, statsMap, formattedInputColumnNames);
+  }
+
+  /**
+   * Adds a row to the parquet buffer.
+   *
+   * @param row row to add
+   * @param out internal buffer to add to
+   * @param statsMap column stats map
+   * @param inputColumnNames list of input column names after formatting
+   * @return row size
+   */
+  private float addRow(
+      Map<String, Object> row,
+      List<List<Object>> out,
+      Map<String, RowBufferStats> statsMap,
+      Set<String> inputColumnNames) {
+    Object[] indexedRow = new Object[fieldIndex.size()];
+    float size = 0F;
+    for (Map.Entry<String, Object> entry : row.entrySet()) {
+      String key = entry.getKey();
+      Object value = entry.getValue();
+      String columnName = formatColumnName(key);
+      int colIndex = fieldIndex.get(columnName).getSecond();
+      RowBufferStats stats = statsMap.get(columnName);
+      ColumnMetadata column = fieldIndex.get(columnName).getFirst();
+      ColumnDescriptor columnDescriptor = schema.getColumns().get(colIndex);
+      PrimitiveType.PrimitiveTypeName typeName =
+          columnDescriptor.getPrimitiveType().getPrimitiveTypeName();
+      ParquetValueParser.ParquetBufferValue valueWithSize =
+          ParquetValueParser.parseColumnValueToParquet(value, column, typeName, stats);
+      indexedRow[colIndex] = valueWithSize.getValue();
+      size += valueWithSize.getSize();
+    }
+    out.add(Arrays.asList(indexedRow));
+
+    for (String columnName : Sets.difference(this.fieldIndex.keySet(), inputColumnNames)) {
+      statsMap.get(columnName).incCurrentNullCount();
+    }
+    return size;
+  }
+
+  @Override
+  void moveTempRowsToActualBuffer(int tempRowCount) {
+    data.addAll(tempData);
+  }
+
+  @Override
+  void clearTempRows() {
+    tempData.clear();
+  }
+
+  @Override
+  boolean hasColumns() {
+    return !fieldIndex.isEmpty();
+  }
+
+  @Override
+  Optional<ParquetChunkData> getSnapshot() {
+    List<List<Object>> oldData = new ArrayList<>();
+    data.forEach(r -> oldData.add(new ArrayList<>(r)));
+    return oldData.isEmpty()
+        ? Optional.empty()
+        : Optional.of(new ParquetChunkData(oldData, metadata));
+  }
+
+  /** Used only for testing. */
+  @Override
+  Object getVectorValueAt(String column, int index) {
+    int colIndex = fieldIndex.get(column).getSecond();
+    Object value = data.get(index).get(colIndex);
+    ColumnMetadata columnMetadata = fieldIndex.get(column).getFirst();
+    String physicalTypeStr = columnMetadata.getPhysicalType();
+    ColumnPhysicalType physicalType = ColumnPhysicalType.valueOf(physicalTypeStr);
+    String logicalTypeStr = columnMetadata.getLogicalType();
+    ColumnLogicalType logicalType = ColumnLogicalType.valueOf(logicalTypeStr);
+    if (logicalType == ColumnLogicalType.FIXED) {
+      if (physicalType == ColumnPhysicalType.SB1) {
+        value = ((Integer) value).byteValue();
+      }
+      if (physicalType == ColumnPhysicalType.SB2) {
+        value = ((Integer) value).shortValue();
+      }
+      if (physicalType == ColumnPhysicalType.SB16) {
+        value = new BigDecimal(new BigInteger((byte[]) value), columnMetadata.getScale());
+      }
+    }
+    if (logicalType == ColumnLogicalType.BINARY && value != null) {
+      value = ((String) value).getBytes(StandardCharsets.UTF_8);
+    }
+    return value;
+  }
+
+  @Override
+  int getTempRowCount() {
+    return tempData.size();
+  }
+
+  @Override
+  void reset() {
+    super.reset();
+    data.clear();
+  }
+
+  @Override
+  public void close(String name) {
+    this.fieldIndex.clear();
+    logger.logInfo(
+        "Trying to close parquet buffer for channel={} from function={}",
+        this.owningChannel.getName(),
+        name);
+  }
+
+  @Override
+  public Flusher<ParquetChunkData> createFlusher(Constants.BdecVersion bdecVerion) {
+    return new ParquetFlusher(schema);
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
new file mode 100644
index 000000000..149ce766c
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
@@ -0,0 +1,292 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+import org.apache.parquet.schema.Types;
+
+/** Generates the Parquet types for the Snowflake's column types */
+public class ParquetTypeGenerator {
+
+  /**
+   * Util class that contains Parquet type and other metadata for that type needed by the Snowflake
+   * server side scanner
+   */
+  static class ParquetTypeInfo {
+    private Type parquetType;
+    private Map<String, String> metadata;
+
+    public Type getParquetType() {
+      return this.parquetType;
+    }
+
+    public Map<String, String> getMetadata() {
+      return this.metadata;
+    }
+
+    public void setParquetType(Type parquetType) {
+      this.parquetType = parquetType;
+    }
+
+    public void setMetadata(Map<String, String> metadata) {
+      this.metadata = metadata;
+    }
+  }
+
+  private static final Set<AbstractRowBuffer.ColumnPhysicalType> TIME_SUPPORTED_PHYSICAL_TYPES =
+      new HashSet<>(
+          Arrays.asList(
+              AbstractRowBuffer.ColumnPhysicalType.SB4, AbstractRowBuffer.ColumnPhysicalType.SB8));
+  private static final Set<AbstractRowBuffer.ColumnPhysicalType>
+      TIMESTAMP_SUPPORTED_PHYSICAL_TYPES =
+          new HashSet<>(
+              Arrays.asList(
+                  AbstractRowBuffer.ColumnPhysicalType.SB8,
+                  AbstractRowBuffer.ColumnPhysicalType.SB16));
+
+  /**
+   * Generate the column parquet type and metadata from the column metadata received from server
+   * side.
+   *
+   * @param column column metadata as received from server side
+   * @param id column id
+   * @return column parquet type
+   */
+  static ParquetTypeInfo generateColumnParquetTypeInfo(ColumnMetadata column, int id) {
+    ParquetTypeInfo res = new ParquetTypeInfo();
+    Type parquetType;
+    Map<String, String> metadata = new HashMap<>();
+    String name = column.getName();
+
+    AbstractRowBuffer.ColumnPhysicalType physicalType;
+    AbstractRowBuffer.ColumnLogicalType logicalType;
+    try {
+      physicalType = AbstractRowBuffer.ColumnPhysicalType.valueOf(column.getPhysicalType());
+      logicalType = AbstractRowBuffer.ColumnLogicalType.valueOf(column.getLogicalType());
+    } catch (IllegalArgumentException e) {
+      throw new SFException(
+          ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
+    }
+
+    metadata.put(Integer.toString(id), logicalType.getOrdinal() + "," + physicalType.getOrdinal());
+
+    // Parquet Type.Repetition in general supports repeated values for the same row column, like a
+    // list of values.
+    // This generator uses only either 0 or 1 value for nullable data type (OPTIONAL: 0 or none
+    // value if it is null)
+    // or exactly 1 value for non-nullable data type (REQUIRED)
+    Type.Repetition repetition =
+        column.getNullable() ? Type.Repetition.OPTIONAL : Type.Repetition.REQUIRED;
+
+    // Handle differently depends on the column logical and physical types
+    switch (logicalType) {
+      case FIXED:
+        parquetType = getFixedColumnParquetType(column, id, physicalType, repetition);
+        break;
+      case ARRAY:
+      case OBJECT:
+      case VARIANT:
+        // mark the column metadata as being an object json for the server side scanner
+        metadata.put(id + ":obj_enc", "1");
+        // parquetType is same as the next one
+      case ANY:
+      case CHAR:
+      case TEXT:
+      case BINARY:
+        parquetType =
+            Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition)
+                .as(LogicalTypeAnnotation.stringType())
+                .id(id)
+                .named(name);
+        break;
+      case TIMESTAMP_LTZ:
+      case TIMESTAMP_NTZ:
+      case TIMESTAMP_TZ:
+        parquetType =
+            getTimeColumnParquetType(
+                column.getScale(),
+                physicalType,
+                logicalType,
+                TIMESTAMP_SUPPORTED_PHYSICAL_TYPES,
+                repetition,
+                id,
+                name);
+        break;
+      case DATE:
+        parquetType =
+            Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition)
+                .as(LogicalTypeAnnotation.dateType())
+                .id(id)
+                .named(name);
+        break;
+      case TIME:
+        parquetType =
+            getTimeColumnParquetType(
+                column.getScale(),
+                physicalType,
+                logicalType,
+                TIME_SUPPORTED_PHYSICAL_TYPES,
+                repetition,
+                id,
+                name);
+        break;
+      case BOOLEAN:
+        parquetType =
+            Types.primitive(PrimitiveType.PrimitiveTypeName.BOOLEAN, repetition).id(id).named(name);
+        break;
+      case REAL:
+        parquetType =
+            Types.primitive(PrimitiveType.PrimitiveTypeName.DOUBLE, repetition).id(id).named(name);
+        break;
+      default:
+        throw new SFException(
+            ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
+    }
+    res.setParquetType(parquetType);
+    res.setMetadata(metadata);
+    return res;
+  }
+
+  /**
+   * Get the parquet type for column of Snowflake FIXED logical type.
+   *
+   * @param column column metadata
+   * @param id column id in Snowflake table schema
+   * @param physicalType Snowflake physical type of column
+   * @param repetition parquet repetition type of column
+   * @return column parquet type
+   */
+  private static Type getFixedColumnParquetType(
+      ColumnMetadata column,
+      int id,
+      AbstractRowBuffer.ColumnPhysicalType physicalType,
+      Type.Repetition repetition) {
+    String name = column.getName();
+    // the LogicalTypeAnnotation.DecimalLogicalTypeAnnotation is used by server side scanner
+    // to discover data type scale and precision
+    LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalLogicalTypeAnnotation =
+        column.getScale() != null && column.getPrecision() != null
+            ? LogicalTypeAnnotation.DecimalLogicalTypeAnnotation.decimalType(
+                column.getScale(), column.getPrecision())
+            : null;
+    Type parquetType;
+    if ((column.getScale() != null && column.getScale() != 0)
+        || physicalType == AbstractRowBuffer.ColumnPhysicalType.SB16) {
+      parquetType =
+          Types.primitive(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, repetition)
+              .length(16)
+              .as(decimalLogicalTypeAnnotation)
+              .id(id)
+              .named(name);
+    } else {
+      switch (physicalType) {
+        case SB1:
+        case SB2:
+        case SB4:
+          parquetType =
+              Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition)
+                  .as(decimalLogicalTypeAnnotation)
+                  .id(id)
+                  .length(4)
+                  .named(name);
+          break;
+        case SB8:
+          parquetType =
+              Types.primitive(PrimitiveType.PrimitiveTypeName.INT64, repetition)
+                  .as(decimalLogicalTypeAnnotation)
+                  .id(id)
+                  .length(8)
+                  .named(name);
+          break;
+        default:
+          throw new SFException(
+              ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
+      }
+    }
+    return parquetType;
+  }
+
+  /**
+   * Get the parquet type for column of a Snowflake time logical type.
+   *
+   * @param scale column scale
+   * @param physicalType Snowflake physical type of column
+   * @param logicalType Snowflake logical type of column
+   * @param supportedPhysicalTypes supported Snowflake physical types for the given column
+   * @param repetition parquet repetition type of column
+   * @param id column id in Snowflake table schema
+   * @param name column name
+   * @return column parquet type
+   */
+  private static Type getTimeColumnParquetType(
+      Integer scale,
+      AbstractRowBuffer.ColumnPhysicalType physicalType,
+      AbstractRowBuffer.ColumnLogicalType logicalType,
+      Set<AbstractRowBuffer.ColumnPhysicalType> supportedPhysicalTypes,
+      Type.Repetition repetition,
+      int id,
+      String name) {
+    if (scale == null || scale > 9 || scale < 0 || !supportedPhysicalTypes.contains(physicalType)) {
+      throw new SFException(
+          ErrorCode.UNKNOWN_DATA_TYPE,
+          "Data type: " + logicalType + ", " + physicalType + ", scale: " + scale);
+    }
+
+    PrimitiveType.PrimitiveTypeName type = getTimePrimitiveType(physicalType);
+    LogicalTypeAnnotation typeAnnotation;
+    int length;
+    switch (physicalType) {
+      case SB4:
+        typeAnnotation = LogicalTypeAnnotation.decimalType(scale, 9);
+        length = 4;
+        break;
+      case SB8:
+        typeAnnotation = LogicalTypeAnnotation.decimalType(scale, 18);
+        length = 8;
+        break;
+      case SB16:
+        typeAnnotation = LogicalTypeAnnotation.decimalType(scale, 38);
+        length = 16;
+        break;
+      default:
+        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
+    }
+    return Types.primitive(type, repetition).as(typeAnnotation).length(length).id(id).named(name);
+  }
+
+  /**
+   * Get the parquet primitive type name for column of a Snowflake time logical type.
+   *
+   * @param physicalType Snowflake physical type of column
+   * @return column parquet primitive type name
+   */
+  private static PrimitiveType.PrimitiveTypeName getTimePrimitiveType(
+      AbstractRowBuffer.ColumnPhysicalType physicalType) {
+    PrimitiveType.PrimitiveTypeName type;
+    switch (physicalType) {
+      case SB4:
+        type = PrimitiveType.PrimitiveTypeName.INT32;
+        break;
+      case SB8:
+        type = PrimitiveType.PrimitiveTypeName.INT64;
+        break;
+      case SB16:
+        type = PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY;
+        break;
+      default:
+        throw new UnsupportedOperationException("Time physical type: " + physicalType);
+    }
+    return type;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
new file mode 100644
index 000000000..671af1a5b
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
@@ -0,0 +1,323 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.math.RoundingMode;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import javax.annotation.Nullable;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.parquet.schema.PrimitiveType;
+
+/** Parses a user column value into Parquet internal representation for buffering. */
+class ParquetValueParser {
+  /** Parquet internal value representation for buffering. */
+  static class ParquetBufferValue {
+    private final Object value;
+    private final float size;
+
+    ParquetBufferValue(Object value, float size) {
+      this.value = value;
+      this.size = size;
+    }
+
+    Object getValue() {
+      return value;
+    }
+
+    float getSize() {
+      return size;
+    }
+  }
+
+  /**
+   * Parses a user column value into Parquet internal representation for buffering.
+   *
+   * @param value column value provided by user in a row
+   * @param columnMetadata column metadata
+   * @param typeName Parquet primitive type name
+   * @param stats column stats to update
+   * @return parsed value and byte size of Parquet internal representation
+   */
+  static ParquetBufferValue parseColumnValueToParquet(
+      Object value,
+      ColumnMetadata columnMetadata,
+      PrimitiveType.PrimitiveTypeName typeName,
+      RowBufferStats stats) {
+    Utils.assertNotNull("Parquet column stats", stats);
+    float size = 0F;
+    if (value != null) {
+      AbstractRowBuffer.ColumnLogicalType logicalType =
+          AbstractRowBuffer.ColumnLogicalType.valueOf(columnMetadata.getLogicalType());
+      AbstractRowBuffer.ColumnPhysicalType physicalType =
+          AbstractRowBuffer.ColumnPhysicalType.valueOf(columnMetadata.getPhysicalType());
+      switch (typeName) {
+        case BOOLEAN:
+          int intValue = DataValidationUtil.validateAndParseBoolean(value);
+          value = intValue > 0;
+          stats.addIntValue(BigInteger.valueOf(intValue));
+          size = 1;
+          break;
+        case INT32:
+          int intVal =
+              getInt32Value(
+                  value,
+                  columnMetadata.getScale(),
+                  Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
+                  logicalType,
+                  physicalType);
+          value = intVal;
+          stats.addIntValue(BigInteger.valueOf(intVal));
+          size = 4;
+          break;
+        case INT64:
+          long longValue =
+              getInt64Value(
+                  value,
+                  columnMetadata.getScale(),
+                  Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
+                  logicalType,
+                  physicalType);
+          value = longValue;
+          stats.addIntValue(BigInteger.valueOf(longValue));
+          size = 8;
+          break;
+        case DOUBLE:
+          double doubleValue = DataValidationUtil.validateAndParseReal(value);
+          value = doubleValue;
+          stats.addRealValue(doubleValue);
+          size = 8;
+          break;
+        case BINARY:
+          String str = getBinaryValue(value, stats, columnMetadata);
+          value = str;
+          size = str.getBytes().length;
+          break;
+        case FIXED_LEN_BYTE_ARRAY:
+          BigInteger intRep =
+              getSb16Value(
+                  value,
+                  columnMetadata.getScale(),
+                  Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
+                  logicalType,
+                  physicalType);
+          stats.addIntValue(intRep);
+          value = getSb16Bytes(intRep);
+          size += 16;
+          break;
+        default:
+          throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
+      }
+    } else {
+      if (!columnMetadata.getNullable()) {
+        throw new SFException(
+            ErrorCode.INVALID_ROW, columnMetadata.getName(), "Passed null to non nullable field");
+      }
+      stats.incCurrentNullCount();
+    }
+    return new ParquetBufferValue(value, size);
+  }
+
+  /**
+   * Parses an int32 value based on Snowflake logical type.
+   *
+   * @param value column value provided by user in a row
+   * @param scale data type scale
+   * @param precision data type precision
+   * @param logicalType Snowflake logical type
+   * @param physicalType Snowflake physical type
+   * @return parsed int32 value
+   */
+  private static int getInt32Value(
+      Object value,
+      @Nullable Integer scale,
+      Integer precision,
+      AbstractRowBuffer.ColumnLogicalType logicalType,
+      AbstractRowBuffer.ColumnPhysicalType physicalType) {
+    int intVal;
+    switch (logicalType) {
+      case DATE:
+        intVal = DataValidationUtil.validateAndParseDate(value);
+        break;
+      case TIME:
+        Utils.assertNotNull("Unexpected null scale for TIME data type", scale);
+        intVal = DataValidationUtil.validateAndParseTime(value, scale).intValue();
+        break;
+      case FIXED:
+        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
+        bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
+        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
+        intVal = bigDecimalValue.intValue();
+        break;
+      default:
+        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
+    }
+    return intVal;
+  }
+
+  /**
+   * Parses an int64 value based on Snowflake logical type.
+   *
+   * @param value column value provided by user in a row
+   * @param scale data type scale
+   * @param precision data type precision
+   * @param logicalType Snowflake logical type
+   * @param physicalType Snowflake physical type
+   * @return parsed int64 value
+   */
+  private static long getInt64Value(
+      Object value,
+      int scale,
+      int precision,
+      AbstractRowBuffer.ColumnLogicalType logicalType,
+      AbstractRowBuffer.ColumnPhysicalType physicalType) {
+    long longValue;
+    switch (logicalType) {
+      case TIME:
+        Utils.assertNotNull("Unexpected null scale for TIME data type", scale);
+        longValue = DataValidationUtil.validateAndParseTime(value, scale).longValue();
+        break;
+      case TIMESTAMP_LTZ:
+      case TIMESTAMP_NTZ:
+        boolean ignoreTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
+        longValue =
+            DataValidationUtil.validateAndParseTimestampNtzSb16(value, scale, ignoreTimezone)
+                .getTimeInScale()
+                .longValue();
+        break;
+      case TIMESTAMP_TZ:
+        longValue =
+            DataValidationUtil.validateAndParseTimestampTz(value, scale)
+                .getSfTimestamp()
+                .orElseThrow(
+                    () ->
+                        new SFException(
+                            ErrorCode.INVALID_ROW,
+                            value,
+                            "Unable to parse timestamp for TIMESTAMP_TZ column"))
+                .toBinary(scale, true)
+                .longValue();
+        break;
+      case FIXED:
+        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
+        bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
+        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
+        longValue = bigDecimalValue.longValue();
+        break;
+      default:
+        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
+    }
+    return longValue;
+  }
+
+  /**
+   * Parses an int128 value based on Snowflake logical type.
+   *
+   * @param value column value provided by user in a row
+   * @param scale data type scale
+   * @param precision data type precision
+   * @param logicalType Snowflake logical type
+   * @param physicalType Snowflake physical type
+   * @return parsed int64 value
+   */
+  private static BigInteger getSb16Value(
+      Object value,
+      int scale,
+      int precision,
+      AbstractRowBuffer.ColumnLogicalType logicalType,
+      AbstractRowBuffer.ColumnPhysicalType physicalType) {
+    switch (logicalType) {
+      case TIMESTAMP_TZ:
+        return DataValidationUtil.validateAndParseTimestampTz(value, scale)
+            .getSfTimestamp()
+            .orElseThrow(
+                () ->
+                    new SFException(
+                        ErrorCode.INVALID_ROW,
+                        value,
+                        "Unable to parse timestamp for TIMESTAMP_TZ column"))
+            .toBinary(scale, true);
+      case TIMESTAMP_LTZ:
+      case TIMESTAMP_NTZ:
+        boolean ignoreTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
+        return DataValidationUtil.validateAndParseTimestampNtzSb16(value, scale, ignoreTimezone)
+            .getTimeInScale();
+      case FIXED:
+        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
+        // explicitly match the BigDecimal input scale with the Snowflake data type scale
+        bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
+        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
+        return bigDecimalValue.unscaledValue();
+      default:
+        throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
+    }
+  }
+
+  /**
+   * Converts an int128 value to its byte array representation.
+   *
+   * @param intRep int128 value
+   * @return byte array representation
+   */
+  static byte[] getSb16Bytes(BigInteger intRep) {
+    byte[] bytes = intRep.toByteArray();
+    byte padByte = (byte) (bytes[0] < 0 ? -1 : 0);
+    byte[] bytesBE = new byte[16];
+    for (int i = 0; i < 16 - bytes.length; i++) {
+      bytesBE[i] = padByte;
+    }
+    System.arraycopy(bytes, 0, bytesBE, 16 - bytes.length, bytes.length);
+    return bytesBE;
+  }
+
+  /**
+   * Converts an object, string or binary value to its byte array representation.
+   *
+   * @param value value to parse
+   * @param stats column stats to update
+   * @param columnMetadata column metadata
+   * @return string (byte array) representation
+   */
+  private static String getBinaryValue(
+      Object value, RowBufferStats stats, ColumnMetadata columnMetadata) {
+    AbstractRowBuffer.ColumnLogicalType logicalType =
+        AbstractRowBuffer.ColumnLogicalType.valueOf(columnMetadata.getLogicalType());
+    String str;
+    if (logicalType.isObject()) {
+      switch (logicalType) {
+        case OBJECT:
+          str = DataValidationUtil.validateAndParseObject(value);
+          break;
+        case VARIANT:
+          str = DataValidationUtil.validateAndParseVariant(value);
+          break;
+        case ARRAY:
+          str = DataValidationUtil.validateAndParseArray(value);
+          break;
+        default:
+          throw new SFException(
+              ErrorCode.UNKNOWN_DATA_TYPE, logicalType, columnMetadata.getPhysicalType());
+      }
+    } else if (logicalType == AbstractRowBuffer.ColumnLogicalType.BINARY) {
+      String maxLengthString = columnMetadata.getLength().toString();
+      byte[] bytes =
+          DataValidationUtil.validateAndParseBinary(
+              value, Optional.of(maxLengthString).map(Integer::parseInt));
+      str = new String(bytes, StandardCharsets.UTF_8);
+      stats.addStrValue(str);
+    } else {
+      String maxLengthString = columnMetadata.getLength().toString();
+      str =
+          DataValidationUtil.validateAndParseString(
+              value, Optional.of(maxLengthString).map(Integer::parseInt));
+      stats.addStrValue(str);
+    }
+    return str;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index 739ae5499..44b90987b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -25,7 +25,8 @@
  * Register one or more blobs to the targeted Snowflake table, it will be done using the dedicated
  * thread in order to maintain ordering per channel
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 class RegisterService<T> {
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
index a229f4db3..2aa9d9861 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
@@ -13,7 +13,8 @@
  * Interface for the buffer in the Streaming Ingest channel that holds the un-flushed rows, these
  * rows will be converted to the underlying format implementation for faster processing
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 interface RowBuffer<T> {
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 82566de02..d8a8bcf21 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -31,7 +31,8 @@
 /**
  * The first version of implementation for SnowflakeStreamingIngestChannel
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIngestChannel {
 
@@ -149,9 +150,20 @@ private RowBuffer<T> createRowBuffer(Constants.BdecVersion bdecVersion) {
     // TODO: The circular dependency SnowflakeStreamingIngestChannelInternal <-> RowBuffer
     // (SNOW-657667)
     // can be probably reconsidered
-    //noinspection unchecked
-    return (RowBuffer<T>)
-        new ArrowRowBuffer((SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>) this);
+    switch (bdecVersion) {
+      case ONE:
+      case TWO:
+        //noinspection unchecked
+        return (RowBuffer<T>)
+            new ArrowRowBuffer((SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>) this);
+      case THREE:
+        //noinspection unchecked
+        return (RowBuffer<T>)
+            new ParquetRowBuffer((SnowflakeStreamingIngestChannelInternal<ParquetChunkData>) this);
+      default:
+        throw new SFException(
+            ErrorCode.INTERNAL_ERROR, "Unsupported BDEC format version: " + bdecVersion);
+    }
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 6a3e99382..8be3c4ed3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -82,7 +82,8 @@
  * <li>the channel cache, which contains all the channels that belong to this account
  * <li>the flush service, which schedules and coordinates the flush to Snowflake tables
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
+ *     ParquetChunkData})
  */
 public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStreamingIngestClient {
 
@@ -553,7 +554,7 @@ List<BlobMetadata> getRetryBlobs(
                   .collect(Collectors.toList());
           if (!relevantChunks.isEmpty()) {
             retryBlobs.add(
-                new BlobMetadata(
+                BlobMetadata.createBlobMetadata(
                     blobMetadata.getPath(),
                     blobMetadata.getMD5(),
                     blobMetadata.getVersion(),
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index fed4022e1..68602e0a9 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -83,7 +83,14 @@ public enum BdecVersion {
     ONE(1),
 
     /** Uses Arrow to generate BDEC chunks with {@link ArrowBatchWriteMode#FILE}. */
-    TWO(2);
+    TWO(2),
+
+    /**
+     * Uses Parquet to generate BDEC chunks with {@link
+     * net.snowflake.ingest.streaming.internal.ParquetRowBuffer} (page-level compression). This
+     * version is experimental and WIP at the moment.
+     */
+    THREE(3);
 
     private final byte version;
 
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 5e5e594cd..9f586a04c 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -14,7 +14,6 @@
 import static net.snowflake.ingest.utils.Constants.USER;
 import static net.snowflake.ingest.utils.Constants.WAREHOUSE;
 import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION;
-import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -35,6 +34,7 @@
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.Utils;
 import org.apache.commons.codec.binary.Base64;
 import org.junit.Assert;
@@ -180,7 +180,7 @@ public static KeyPair getKeyPair() throws Exception {
     return keyPair;
   }
 
-  public static Properties getProperties() throws Exception {
+  public static Properties getProperties(Constants.BdecVersion bdecVersion) throws Exception {
     if (profile == null) {
       init();
     }
@@ -195,16 +195,10 @@ public static Properties getProperties() throws Exception {
     props.put(PRIVATE_KEY, privateKeyPem);
     props.put(ROLE, role);
     props.put(ACCOUNT_URL, getAccountURL());
-    props.put(BLOB_FORMAT_VERSION, getBlobFormatVersion());
+    props.put(BLOB_FORMAT_VERSION, bdecVersion.toByte());
     return props;
   }
 
-  private static byte getBlobFormatVersion() {
-    return profile.has(BLOB_FORMAT_VERSION)
-        ? (byte) profile.get(BLOB_FORMAT_VERSION).asInt()
-        : BLOB_FORMAT_VERSION_DEFAULT.toByte();
-  }
-
   /**
    * Create snowflake jdbc connection
    *
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
index c07966c68..5785bd221 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
@@ -57,15 +57,13 @@ public void testFieldNumberAfterFlush() {
   @Test
   public void buildFieldFixedSB1() {
     // FIXED, SB1
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB1");
-    testCol.setNullable(true);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB1")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -80,15 +78,13 @@ public void buildFieldFixedSB1() {
 
   @Test
   public void buildFieldFixedSB2() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB2");
-    testCol.setNullable(false);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB2")
+            .nullable(false)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -103,15 +99,13 @@ public void buildFieldFixedSB2() {
 
   @Test
   public void buildFieldFixedSB4() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB4");
-    testCol.setNullable(true);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB4")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -125,15 +119,13 @@ public void buildFieldFixedSB4() {
 
   @Test
   public void buildFieldFixedSB8() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -147,15 +139,13 @@ public void buildFieldFixedSB8() {
 
   @Test
   public void buildFieldFixedSB16() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB16");
-    testCol.setNullable(true);
-    testCol.setLogicalType("FIXED");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     ArrowType expectedType =
@@ -172,15 +162,13 @@ public void buildFieldFixedSB16() {
 
   @Test
   public void buildFieldLobVariant() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("LOB");
-    testCol.setNullable(true);
-    testCol.setLogicalType("VARIANT");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("VARIANT")
+            .physicalType("LOB")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -194,15 +182,13 @@ public void buildFieldLobVariant() {
 
   @Test
   public void buildFieldTimestampNtzSB8() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIMESTAMP_NTZ");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -216,15 +202,13 @@ public void buildFieldTimestampNtzSB8() {
 
   @Test
   public void buildFieldTimestampNtzSB16() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB16");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIMESTAMP_NTZ");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -242,15 +226,13 @@ public void buildFieldTimestampNtzSB16() {
 
   @Test
   public void buildFieldTimestampTzSB8() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIMESTAMP_TZ");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_TZ")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -268,15 +250,13 @@ public void buildFieldTimestampTzSB8() {
 
   @Test
   public void buildFieldTimestampTzSB16() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB16");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIMESTAMP_TZ");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_TZ")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -295,16 +275,14 @@ public void buildFieldTimestampTzSB16() {
   }
 
   @Test
-  public void buildFieldDate() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("DATE");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+  public void buildFieldTimestampDate() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("DATE")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -318,15 +296,13 @@ public void buildFieldDate() {
 
   @Test
   public void buildFieldTimeSB4() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB4");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIME");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB4")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -340,15 +316,13 @@ public void buildFieldTimeSB4() {
 
   @Test
   public void buildFieldTimeSB8() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB8");
-    testCol.setNullable(true);
-    testCol.setLogicalType("TIME");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -362,15 +336,13 @@ public void buildFieldTimeSB8() {
 
   @Test
   public void buildFieldBoolean() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("BINARY");
-    testCol.setNullable(true);
-    testCol.setLogicalType("BOOLEAN");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("BOOLEAN")
+            .physicalType("BINARY")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
@@ -384,15 +356,13 @@ public void buildFieldBoolean() {
 
   @Test
   public void buildFieldRealSB16() {
-    ColumnMetadata testCol = new ColumnMetadata();
-    testCol.setName("testCol");
-    testCol.setPhysicalType("SB16");
-    testCol.setNullable(true);
-    testCol.setLogicalType("REAL");
-    testCol.setByteLength(14);
-    testCol.setLength(11);
-    testCol.setScale(0);
-    testCol.setPrecision(4);
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("REAL")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
     Assert.assertEquals("testCol", result.getName());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java b/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
new file mode 100644
index 000000000..51d4226bb
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
@@ -0,0 +1,164 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+/** Helper class to build ColumnMetadata */
+public class ColumnMetadataBuilder {
+  private String name;
+  private String type;
+  private String logicalType;
+  private String physicalType;
+  private Integer precision;
+  private Integer scale;
+  private Integer byteLength;
+  private Integer length;
+  private boolean nullable;
+  private String collation;
+
+  /**
+   * Returns a new ColumnMetadata builder
+   *
+   * @return builder
+   */
+  public static ColumnMetadataBuilder newBuilder() {
+    ColumnMetadataBuilder mb = new ColumnMetadataBuilder();
+    mb.name = "testCol";
+    mb.byteLength = 14;
+    mb.length = 11;
+    mb.scale = 0;
+    mb.precision = 4;
+    return mb;
+  }
+
+  /**
+   * Set name
+   *
+   * @param name column name
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder name(String name) {
+    this.name = name;
+    return this;
+  }
+
+  /**
+   * Set type
+   *
+   * @param type type (as defined in ColumnMetadata)
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder type(String type) {
+    this.type = type;
+    return this;
+  }
+
+  /**
+   * Set column logical type
+   *
+   * @param logicalType logical type
+   * @return columnMetadata object
+   */
+  public ColumnMetadataBuilder logicalType(String logicalType) {
+    this.logicalType = logicalType;
+    return this;
+  }
+
+  /**
+   * Set column physical type
+   *
+   * @param physicalType physical type
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder physicalType(String physicalType) {
+    this.physicalType = physicalType;
+    return this;
+  }
+
+  /**
+   * Set column precision
+   *
+   * @param precision precision
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder precision(int precision) {
+    this.precision = precision;
+    return this;
+  }
+
+  /**
+   * Set column scale
+   *
+   * @param scale scale
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder scale(int scale) {
+    this.scale = scale;
+    return this;
+  }
+
+  /**
+   * Set column length in bytes
+   *
+   * @param byteLength length in bytes
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder byteLength(int byteLength) {
+    this.byteLength = byteLength;
+    return this;
+  }
+
+  /**
+   * Set column length (in chars)
+   *
+   * @param length length
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder length(int length) {
+    this.length = length;
+    return this;
+  }
+
+  /**
+   * Set column nullability
+   *
+   * @param nullable nullable
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder nullable(boolean nullable) {
+    this.nullable = nullable;
+    return this;
+  }
+
+  /**
+   * Set column collation
+   *
+   * @param collation collation
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder collation(String collation) {
+    this.collation = collation;
+    return this;
+  }
+
+  /**
+   * Build a columnMetadata object from the set values
+   *
+   * @return columnMetadata object
+   */
+  public ColumnMetadata build() {
+    ColumnMetadata colMetadata = new ColumnMetadata();
+    colMetadata.setName(name);
+    colMetadata.setType(type);
+    colMetadata.setPhysicalType(physicalType);
+    colMetadata.setNullable(nullable);
+    colMetadata.setLogicalType(logicalType);
+    colMetadata.setByteLength(byteLength);
+    colMetadata.setLength(length);
+    colMetadata.setScale(scale);
+    colMetadata.setPrecision(precision);
+    colMetadata.setCollation(collation);
+    return colMetadata;
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 80103e2df..540cd5ea8 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -55,7 +55,8 @@
 public class FlushServiceTest {
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> testContextFactory() {
-    return Arrays.asList(new Object[][] {{ArrowTestContext.createFactory()}});
+    return Arrays.asList(
+        new Object[][] {{ArrowTestContext.createFactory()}, {ParquetTestContext.createFactory()}});
   }
 
   public FlushServiceTest(TestContextFactory<?> testContextFactory) {
@@ -182,11 +183,6 @@ ChannelBuilder setEncryptionKeyId(Long encryptionKeyId) {
         return this;
       }
 
-      ChannelBuilder setOnErrorOption(OpenChannelRequest.OnErrorOption onErrorOption) {
-        this.onErrorOption = onErrorOption;
-        return this;
-      }
-
       SnowflakeStreamingIngestChannelInternal<T> buildAndAdd() {
         SnowflakeStreamingIngestChannelInternal<T> channel =
             createChannel(
@@ -285,6 +281,48 @@ TestContext<VectorSchemaRoot> create() {
     }
   }
 
+  private static class ParquetTestContext extends TestContext<List<List<Object>>> {
+
+    SnowflakeStreamingIngestChannelInternal<List<List<Object>>> createChannel(
+        String name,
+        String dbName,
+        String schemaName,
+        String tableName,
+        String offsetToken,
+        Long channelSequencer,
+        Long rowSequencer,
+        String encryptionKey,
+        Long encryptionKeyId,
+        OpenChannelRequest.OnErrorOption onErrorOption) {
+      return new SnowflakeStreamingIngestChannelInternal<>(
+          name,
+          dbName,
+          schemaName,
+          tableName,
+          offsetToken,
+          channelSequencer,
+          rowSequencer,
+          client,
+          encryptionKey,
+          encryptionKeyId,
+          onErrorOption,
+          Constants.BdecVersion.THREE,
+          null);
+    }
+
+    @Override
+    public void close() {}
+
+    static TestContextFactory<List<List<Object>>> createFactory() {
+      return new TestContextFactory<List<List<Object>>>("Parquet") {
+        @Override
+        TestContext<List<List<Object>>> create() {
+          return new ParquetTestContext();
+        }
+      };
+    }
+  }
+
   TestContextFactory<?> testContextFactory;
   TestContext<?> testContext;
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
new file mode 100644
index 000000000..a477ac977
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
@@ -0,0 +1,480 @@
+package net.snowflake.ingest.streaming.internal;
+
+import java.util.Map;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.PrimitiveType;
+import org.apache.parquet.schema.Type;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ParquetTypeGeneratorTest {
+
+  @Test
+  public void buildFieldFixedSB1() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB1")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(4)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
+        .expectedLogicalTypeAnnotation(
+            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB1.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldFixedSB2() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB2")
+            .nullable(false)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(4)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
+        .expectedLogicalTypeAnnotation(
+            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
+        .expectedRepetition(Type.Repetition.REQUIRED)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB2.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldFixedSB4() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB4")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(4)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
+        .expectedLogicalTypeAnnotation(
+            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB4.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldFixedSB8() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(8)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
+        .expectedLogicalTypeAnnotation(
+            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldFixedSB16() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(16)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+        .expectedLogicalTypeAnnotation(
+            LogicalTypeAnnotation.decimalType(testCol.getScale(), testCol.getPrecision()))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.FIXED.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldLobVariant() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("VARIANT")
+            .physicalType("LOB")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(0)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.BINARY)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.stringType())
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.VARIANT.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.LOB.getOrdinal())
+        .assertMatches();
+
+    Assert.assertEquals("1", typeInfo.getMetadata().get(0 + ":obj_enc"));
+  }
+
+  @Test
+  public void buildFieldTimestampNtzSB8() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(8)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldTimestampNtzSB16() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(16)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 38))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldTimestampTzSB8() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_TZ")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(8)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_TZ.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldTimestampTzSB16() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_TZ")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(16)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 38))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_TZ.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldDate() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("DATE")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(0)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.dateType())
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.DATE.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldTimeSB4() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB4")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(4)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 9))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIME.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB4.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldTimeSB8() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB8")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(8)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
+        .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.TIME.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB8.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldBoolean() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("BOOLEAN")
+            .physicalType("BINARY")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(0)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.BOOLEAN)
+        .expectedLogicalTypeAnnotation(null)
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.BOOLEAN.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.BINARY.getOrdinal())
+        .assertMatches();
+  }
+
+  @Test
+  public void buildFieldRealSB16() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("REAL")
+            .physicalType("SB16")
+            .nullable(true)
+            .build();
+
+    ParquetTypeGenerator.ParquetTypeInfo typeInfo =
+        ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
+    ParquetTypeInfoAssertionBuilder.newBuilder()
+        .typeInfo(typeInfo)
+        .expectedFieldName("testCol")
+        .expectedFieldId(0)
+        .expectedTypeLength(0)
+        .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.DOUBLE)
+        .expectedLogicalTypeAnnotation(null)
+        .expectedRepetition(Type.Repetition.OPTIONAL)
+        .expectedColMetaData(
+            AbstractRowBuffer.ColumnLogicalType.REAL.getOrdinal()
+                + ","
+                + AbstractRowBuffer.ColumnPhysicalType.SB16.getOrdinal())
+        .assertMatches();
+  }
+
+  /** Builder that helps to assert parquet type info */
+  private static class ParquetTypeInfoAssertionBuilder {
+    private String fieldName;
+    private int fieldId;
+    private PrimitiveType.PrimitiveTypeName primitiveTypeName;
+    private LogicalTypeAnnotation logicalTypeAnnotation;
+    private Type.Repetition repetition;
+    private int typeLength;
+    private String colMetadata;
+    private ParquetTypeGenerator.ParquetTypeInfo typeInfo;
+
+    static ParquetTypeInfoAssertionBuilder newBuilder() {
+      ParquetTypeInfoAssertionBuilder builder = new ParquetTypeInfoAssertionBuilder();
+      return builder;
+    }
+
+    ParquetTypeInfoAssertionBuilder typeInfo(ParquetTypeGenerator.ParquetTypeInfo typeInfo) {
+      this.typeInfo = typeInfo;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedFieldName(String fieldName) {
+      this.fieldName = fieldName;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedFieldId(int fieldId) {
+      this.fieldId = fieldId;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedPrimitiveTypeName(
+        PrimitiveType.PrimitiveTypeName primitiveTypeName) {
+      this.primitiveTypeName = primitiveTypeName;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedLogicalTypeAnnotation(
+        LogicalTypeAnnotation logicalTypeAnnotation) {
+      this.logicalTypeAnnotation = logicalTypeAnnotation;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedRepetition(Type.Repetition repetition) {
+      this.repetition = repetition;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedTypeLength(int typeLength) {
+      this.typeLength = typeLength;
+      return this;
+    }
+
+    ParquetTypeInfoAssertionBuilder expectedColMetaData(String colMetaData) {
+      this.colMetadata = colMetaData;
+      return this;
+    }
+
+    void assertMatches() {
+      Type type = typeInfo.getParquetType();
+      Map<String, String> metadata = typeInfo.getMetadata();
+      Assert.assertEquals(fieldName, type.getName());
+      Assert.assertEquals(typeLength, type.asPrimitiveType().getTypeLength());
+      Assert.assertEquals(fieldId, type.asPrimitiveType().getId().intValue());
+
+      Assert.assertEquals(primitiveTypeName, type.asPrimitiveType().getPrimitiveTypeName());
+      Assert.assertEquals(logicalTypeAnnotation, type.getLogicalTypeAnnotation());
+      Assert.assertEquals(repetition, type.getRepetition());
+      Assert.assertEquals(colMetadata, metadata.get(type.getId().toString()));
+    }
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
new file mode 100644
index 000000000..3dc313dc6
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
@@ -0,0 +1,570 @@
+package net.snowflake.ingest.streaming.internal;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.Map;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.parquet.schema.PrimitiveType;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ParquetValueParserTest {
+
+  @Test
+  public void parseValueFixedSB1ToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB1")
+            .scale(0)
+            .precision(2)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            12, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Integer.class)
+        .expectedParsedValue(12)
+        .expectedSize(4.0f)
+        .expectedMinMax(BigInteger.valueOf(12))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueFixedSB2ToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB2")
+            .scale(0)
+            .precision(4)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            1234, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Integer.class)
+        .expectedParsedValue(1234)
+        .expectedSize(4.0f)
+        .expectedMinMax(BigInteger.valueOf(1234))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueFixedSB4ToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB4")
+            .scale(0)
+            .precision(9)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            123456789, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Integer.class)
+        .expectedParsedValue(123456789)
+        .expectedSize(4.0f)
+        .expectedMinMax(BigInteger.valueOf(123456789))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueFixedSB8ToInt64() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB8")
+            .scale(0)
+            .precision(18)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            123456789987654321L, testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Long.class)
+        .expectedParsedValue(123456789987654321L)
+        .expectedSize(8.0f)
+        .expectedMinMax(BigInteger.valueOf(123456789987654321L))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueFixedSB16ToByteArray() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB16")
+            .scale(0)
+            .precision(38)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            new BigDecimal("91234567899876543219876543211234567891"),
+            testCol,
+            PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
+            rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(byte[].class)
+        .expectedParsedValue(
+            ParquetValueParser.getSb16Bytes(
+                new BigInteger("91234567899876543219876543211234567891")))
+        .expectedSize(16.0f)
+        .expectedMinMax(new BigInteger("91234567899876543219876543211234567891"))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueFixedDecimalToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("FIXED")
+            .physicalType("SB8")
+            .scale(5)
+            .precision(10)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            new BigDecimal("12345.54321"),
+            testCol,
+            PrimitiveType.PrimitiveTypeName.DOUBLE,
+            rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Double.class)
+        .expectedParsedValue(Double.valueOf("12345.54321"))
+        .expectedSize(8.0f)
+        .expectedMinMax(Double.valueOf("12345.54321"))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueDouble() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("REAL")
+            .physicalType("DOUBLE")
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            12345.54321d, testCol, PrimitiveType.PrimitiveTypeName.DOUBLE, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Double.class)
+        .expectedParsedValue(Double.valueOf(12345.54321))
+        .expectedSize(8.0f)
+        .expectedMinMax(Double.valueOf(12345.54321))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueBoolean() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("BOOLEAN")
+            .physicalType("SB1")
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            true, testCol, PrimitiveType.PrimitiveTypeName.BOOLEAN, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Boolean.class)
+        .expectedParsedValue(true)
+        .expectedSize(1.0f)
+        .expectedMinMax(BigInteger.valueOf(1))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueBinary() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("BINARY")
+            .physicalType("LOB")
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "Length7".getBytes(), testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(String.class)
+        .expectedParsedValue("Length7")
+        .expectedSize(7.0f)
+        .expectedMinMax("Length7")
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueVariantToBinary() {
+    testJsonWithLogicalType("VARIANT");
+  }
+
+  @Test
+  public void parseValueObjectToBinary() {
+    testJsonWithLogicalType("OBJECT");
+  }
+
+  private void testJsonWithLogicalType(String logicalType) {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType(logicalType)
+            .physicalType("BINARY")
+            .nullable(true)
+            .build();
+
+    String var =
+        "{\"key1\":-879869596,\"key2\":\"value2\",\"key3\":null,"
+            + "\"key4\":{\"key41\":0.032437,\"key42\":\"value42\",\"key43\":null}}";
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            var, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(String.class)
+        .expectedParsedValue(var)
+        .expectedSize(var.getBytes().length)
+        .expectedMinMax(null)
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueArrayToBinary() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("ARRAY")
+            .physicalType("BINARY")
+            .nullable(true)
+            .build();
+
+    Map<String, String> input = new HashMap<>();
+    input.put("a", "1");
+    input.put("b", "2");
+    input.put("c", "3");
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            input, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
+
+    String resultArray = "[{\"a\":\"1\",\"b\":\"2\",\"c\":\"3\"}]";
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(String.class)
+        .expectedParsedValue(resultArray)
+        .expectedSize(resultArray.length())
+        .expectedMinMax(null)
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueTimestampNtzSB4Error() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB4")
+            .scale(0) // seconds
+            .precision(9)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    SFException exception =
+        Assert.assertThrows(
+            SFException.class,
+            () ->
+                ParquetValueParser.parseColumnValueToParquet(
+                    "2013-04-28 20:57:00",
+                    testCol,
+                    PrimitiveType.PrimitiveTypeName.INT32,
+                    rowBufferStats));
+    Assert.assertEquals(
+        "Unknown data type for logical: TIMESTAMP_NTZ, physical: SB4.", exception.getMessage());
+  }
+
+  @Test
+  public void parseValueTimestampNtzSB8ToINT64() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB8")
+            .scale(3) // millis
+            .precision(18)
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "2013-04-28 20:57:01.000",
+            testCol,
+            PrimitiveType.PrimitiveTypeName.INT64,
+            rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Long.class)
+        .expectedParsedValue(1367182621000L)
+        .expectedSize(8.0f)
+        .expectedMinMax(BigInteger.valueOf(1367182621000L))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueTimestampNtzSB16ToByteArray() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIMESTAMP_NTZ")
+            .physicalType("SB16")
+            .nullable(true)
+            .scale(9) // nanos
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "2022-09-18 22:05:07.123456789",
+            testCol,
+            PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
+            rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(byte[].class)
+        .expectedParsedValue(
+            ParquetValueParser.getSb16Bytes(BigInteger.valueOf(1663538707123456789L)))
+        .expectedSize(16.0f)
+        .expectedMinMax(BigInteger.valueOf(1663538707123456789L))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueDateToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("DATE")
+            .physicalType("SB4")
+            .scale(0) // seconds
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "2021-01-01", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Integer.class)
+        .expectedParsedValue(Integer.valueOf(18628))
+        .expectedSize(4.0f)
+        .expectedMinMax(BigInteger.valueOf(18628))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueTimeSB4ToInt32() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB4")
+            .scale(0) // seconds
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "01:00:00", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Integer.class)
+        .expectedParsedValue(3600)
+        .expectedSize(4.0f)
+        .expectedMinMax(BigInteger.valueOf(3600))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueTimeSB8ToInt64() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB8")
+            .scale(3) // milliseconds
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            "01:00:00.123", testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(Long.class)
+        .expectedParsedValue(3600123L)
+        .expectedSize(8.0f)
+        .expectedMinMax(BigInteger.valueOf(3600123))
+        .assertMatches();
+  }
+
+  @Test
+  public void parseValueTimeSB16Error() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TIME")
+            .physicalType("SB16")
+            .scale(9) // nanos
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats();
+    SFException exception =
+        Assert.assertThrows(
+            SFException.class,
+            () ->
+                ParquetValueParser.parseColumnValueToParquet(
+                    "11:00:00.12345678",
+                    testCol,
+                    PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
+                    rowBufferStats));
+    Assert.assertEquals(
+        "Unknown data type for logical: TIME, physical: SB16.", exception.getMessage());
+  }
+
+  /** Builder that helps to assert parsing of values to parquet types */
+  private static class ParquetValueParserAssertionBuilder {
+    private ParquetValueParser.ParquetBufferValue parquetBufferValue;
+    private RowBufferStats rowBufferStats;
+    private Class valueClass;
+    private Object value;
+    private float size;
+    private Object minMaxStat;
+
+    static ParquetValueParserAssertionBuilder newBuilder() {
+      ParquetValueParserAssertionBuilder builder = new ParquetValueParserAssertionBuilder();
+      return builder;
+    }
+
+    ParquetValueParserAssertionBuilder parquetBufferValue(
+        ParquetValueParser.ParquetBufferValue parquetBufferValue) {
+      this.parquetBufferValue = parquetBufferValue;
+      return this;
+    }
+
+    ParquetValueParserAssertionBuilder rowBufferStats(RowBufferStats rowBufferStats) {
+      this.rowBufferStats = rowBufferStats;
+      return this;
+    }
+
+    ParquetValueParserAssertionBuilder expectedValueClass(Class valueClass) {
+      this.valueClass = valueClass;
+      return this;
+    }
+
+    ParquetValueParserAssertionBuilder expectedParsedValue(Object value) {
+      this.value = value;
+      return this;
+    }
+
+    ParquetValueParserAssertionBuilder expectedSize(float size) {
+      this.size = size;
+      return this;
+    }
+
+    public ParquetValueParserAssertionBuilder expectedMinMax(Object minMaxStat) {
+      this.minMaxStat = minMaxStat;
+      return this;
+    }
+
+    void assertMatches() {
+      Assert.assertEquals(valueClass, parquetBufferValue.getValue().getClass());
+      System.out.println("parquetBufferValue = " + parquetBufferValue.getValue().toString());
+      if (valueClass.equals(byte[].class)) {
+        Assert.assertArrayEquals((byte[]) value, (byte[]) parquetBufferValue.getValue());
+      } else {
+        Assert.assertEquals(value, parquetBufferValue.getValue());
+      }
+      Assert.assertEquals(size, parquetBufferValue.getSize(), 0);
+      if (minMaxStat instanceof BigInteger) {
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinIntValue());
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxIntValue());
+        return;
+      } else if (minMaxStat instanceof String || valueClass.equals(String.class)) {
+        // String can have null min/max stats for variant data types
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinColStrValue());
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxColStrValue());
+        return;
+      } else if (minMaxStat instanceof Double || minMaxStat instanceof BigDecimal) {
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinRealValue());
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxRealValue());
+        return;
+      }
+      throw new IllegalArgumentException("Unknown data type for min stat");
+    }
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 7bd986e8a..8d28c0cb9 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -26,7 +26,11 @@
 public class RowBufferTest {
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> bdecVersion() {
-    return Arrays.asList(new Object[][] {{"Arrow", Constants.BdecVersion.ONE}});
+    return Arrays.asList(
+        new Object[][] {
+          {"Arrow", Constants.BdecVersion.ONE},
+          {"Parquet", Constants.BdecVersion.THREE}
+        });
   }
 
   private final Constants.BdecVersion bdecVersion;
@@ -1029,6 +1033,7 @@ private void testE2EBinaryHelper(OpenChannelRequest.OnErrorOption onErrorOption)
     colBinary.setPhysicalType("LOB");
     colBinary.setNullable(true);
     colBinary.setLogicalType("BINARY");
+    colBinary.setLength(32);
     colBinary.setScale(0);
 
     innerBuffer.setupSchema(Collections.singletonList(colBinary));
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 175db5fb8..c5189bcde 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -9,14 +9,15 @@
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
-import java.util.Random;
 import java.util.TimeZone;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
@@ -31,6 +32,7 @@
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.ParameterProvider;
 import net.snowflake.ingest.utils.SFException;
@@ -38,9 +40,23 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 /** Example streaming ingest sdk integration test */
+@RunWith(Parameterized.class)
 public class StreamingIngestIT {
+  @Parameterized.Parameters(name = "{0}")
+  public static Collection<Object[]> bdecVersion() {
+    return Arrays.asList(
+        new Object[][] {
+          {"Arrow", Constants.BdecVersion.ONE},
+          // TODO: uncomment once SNOW-659721 is deployed and we set the parameter
+          // DISABLE_PARQUET_CACHE to true for the test account
+          // {"Parquet", Constants.BdecVersion.THREE}
+        });
+  }
+
   private static final String TEST_TABLE = "STREAMING_INGEST_TEST_TABLE";
   private static final String TEST_DB_PREFIX = "STREAMING_INGEST_TEST_DB";
   private static final String TEST_SCHEMA = "STREAMING_INGEST_TEST_SCHEMA";
@@ -55,11 +71,23 @@ public class StreamingIngestIT {
   private Connection jdbcConnection;
   private String testDb;
 
+  private final Constants.BdecVersion bdecVersion;
+
+  public StreamingIngestIT(
+      @SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
+    this.bdecVersion = bdecVersion;
+  }
+
   @Before
   public void beforeAll() throws Exception {
     testDb = TEST_DB_PREFIX + "_" + UUID.randomUUID().toString().substring(0, 4);
     // Create a streaming ingest client
     jdbcConnection = TestUtils.getConnection(true);
+    if (bdecVersion == Constants.BdecVersion.THREE) {
+      // TODO: encryption and interleaved mode are not yet supported by server side's Parquet
+      // scanner if local file cache is enabled (SNOW-656500)
+      jdbcConnection.createStatement().execute("alter session set disable_parquet_cache=true;");
+    }
     jdbcConnection
         .createStatement()
         .execute(String.format("create or replace database %s;", testDb));
@@ -78,7 +106,7 @@ public void beforeAll() throws Exception {
         .createStatement()
         .execute(String.format("use warehouse %s", TestUtils.getWarehouse()));
 
-    prop = TestUtils.getProperties();
+    prop = TestUtils.getProperties(bdecVersion);
     if (prop.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       prop.setProperty(ROLE, "ACCOUNTADMIN");
     }
@@ -1042,7 +1070,7 @@ private void verifyTableRowCount(int rowNumber, String tableName) {
   }
 
   @Test
-  public void testDataTypes() {
+  public void testDataTypes() throws SQLException, ParseException {
     String tableName = "t_data_types";
     try {
       jdbcConnection
@@ -1080,94 +1108,195 @@ public void testDataTypes() {
 
     SnowflakeStreamingIngestChannel channel = openChannel(tableName, "CHANNEL");
 
-    Random r = new Random();
-    for (int val = 0; val < 10000; val++) {
-      verifyInsertValidationResponse(channel.insertRow(getRandomRow(r), Integer.toString(val)));
-    }
-    waitChannelFlushed(channel, 10000);
+    Map<String, Object> negRow = getNegRow();
+    verifyInsertValidationResponse(channel.insertRow(negRow, Integer.toString(0)));
+
+    Map<String, Object> nullRow = getNullRow();
+    verifyInsertValidationResponse(channel.insertRow(nullRow, Integer.toString(1)));
+
+    Map<String, Object> posRow = getPosRow();
+    verifyInsertValidationResponse(channel.insertRow(posRow, Integer.toString(2)));
+
+    waitChannelFlushed(channel, 3);
+    verifyTableRowCount(3, tableName);
+
+    ResultSet result =
+        jdbcConnection
+            .createStatement()
+            .executeQuery(
+                String.format(
+                    "select * from %s.%s.%s order by num", testDb, TEST_SCHEMA, tableName));
+
+    result.next(); // negRow
+    assertNegRow(negRow, result);
+
+    result.next(); // nullRow
+    assertNullRow(nullRow, result);
+
+    result.next(); // posRow
+    assertPosRow(posRow, result);
   }
 
-  private static Map<String, Object> getRandomRow(Random r) {
-    Map<String, Object> row = new HashMap<>();
-    row.put("num", r.nextInt());
-    row.put("num_10_5", nextFloat(r));
-    row.put(
-        "num_38_0",
-        new BigDecimal("" + nextLongOfPrecision(r, 18) + Math.abs(nextLongOfPrecision(r, 18))));
-    row.put("num_2_0", r.nextInt(100));
-    row.put("num_4_0", r.nextInt(10000));
-    row.put("num_9_0", r.nextInt(1000000000));
-    row.put("num_18_0", nextLongOfPrecision(r, 18));
-    row.put("num_float", nextFloat(r));
-    row.put("str_varchar", nextString(r));
-    row.put("bin", nextBytes(r));
-    row.put("bl", r.nextBoolean());
-    row.put("var", nextJson(r));
-    row.put("obj", nextJson(r));
-    row.put("arr", Arrays.asList(r.nextInt(100), r.nextInt(100), r.nextInt(100)));
-    row.put(
-        "epochdays",
-        String.valueOf(Math.abs(r.nextInt()) % (18963 * 24 * 60 * 60))); // DATE, 02.12.2021
-    row.put(
-        "timesec",
-        String.valueOf(
-            (r.nextInt(11) * 60 * 60 + r.nextInt(59) * 60 + r.nextInt(59)) * 10000
-                + r.nextInt(9999))); // TIME(4), 05:12:43.4536
-    row.put(
-        "timenano",
-        String.valueOf(
-            (14 * 60 * 60 + 26 * 60 + 34) * 1000000000L
-                + 437582643)); // TIME(9), 14:26:34.437582643
-    row.put(
-        "epochsec",
-        String.valueOf(
-            Math.abs(r.nextLong()) % 1638459438)); // TIMESTAMP_LTZ(0), 02.12.2021 15:37:18
-    row.put(
-        "epochnano",
-        String.format(
-            "%d%d",
-            Math.abs(r.nextInt()) % 1999999999,
-            100000000
-                + Math.abs(
-                    r.nextInt(899999999)))); // TIMESTAMP_LTZ(9), 18.05.2033 03:33:19.999999999
-    return row;
+  private Map<String, Object> getNegRow() {
+    Map<String, Object> negRow = new HashMap<>();
+    negRow.put("num", -123);
+    negRow.put("num_10_5", new BigDecimal("-12345.54321"));
+    negRow.put("num_38_0", new BigDecimal("-91234567899876543219876543211234567891"));
+    negRow.put("num_2_0", -12);
+    negRow.put("num_4_0", -1234);
+    negRow.put("num_9_0", -123456789);
+    negRow.put("num_18_0", -123456789987654321L);
+    negRow.put("num_float", -12345.54321f);
+    negRow.put("str_varchar", "zxccvbnm,.");
+    negRow.put("bin", new byte[] {1, 2, 3});
+    negRow.put("bl", true);
+    negRow.put("var", "{}");
+    negRow.put("obj", "{}");
+    negRow.put("arr", Arrays.asList());
+    negRow.put("epochdays", "0");
+    negRow.put("epochsec", "0");
+    negRow.put("epochnano", "0");
+    negRow.put("timesec", "0");
+    negRow.put("timenano", "0");
+
+    return negRow;
   }
 
-  private static long nextLongOfPrecision(Random r, int precision) {
-    return r.nextLong() % Math.round(Math.pow(10, precision));
+  private Map<String, Object> getNullRow() {
+    Map<String, Object> nullRow = new HashMap<>();
+    nullRow.put("num", 0);
+    nullRow.put("num_10_5", new BigDecimal("0.00000"));
+    nullRow.put("num_38_0", new BigDecimal("0"));
+    nullRow.put("num_2_0", 0);
+    nullRow.put("num_4_0", 0);
+    nullRow.put("num_9_0", 0);
+    nullRow.put("num_18_0", 0L);
+    nullRow.put("num_float", 0.0f);
+    nullRow.put("str_varchar", null);
+    nullRow.put("bin", null);
+    nullRow.put("bl", false);
+    nullRow.put("var", null);
+    nullRow.put("obj", null);
+    nullRow.put("arr", null);
+    nullRow.put("epochdays", null);
+    nullRow.put("epochsec", null);
+    nullRow.put("epochnano", null);
+    nullRow.put("timesec", null);
+    nullRow.put("timenano", null);
+
+    return nullRow;
   }
 
-  private static String nextString(Random r) {
-    return new String(nextBytes(r));
+  private Map<String, Object> getPosRow() {
+    Map<String, Object> posRow = new HashMap<>();
+    posRow.put("num", 123);
+    posRow.put("num_10_5", new BigDecimal("12345.54321"));
+    posRow.put("num_38_0", new BigDecimal("91234567899876543219876543211234567891"));
+    posRow.put("num_2_0", 12);
+    posRow.put("num_4_0", 1234);
+    posRow.put("num_9_0", 123456789);
+    posRow.put("num_18_0", 123456789987654321L);
+    posRow.put("num_float", 12345.54321f);
+    posRow.put("str_varchar", "abcd123456.");
+    posRow.put("bin", new byte[] {4, 5, 6});
+    posRow.put("bl", true);
+    posRow.put(
+        "var",
+        "{ \"a\": 1, \"b\": \"qwerty\", \"c\": null, \"d\": { \"e\": 2, \"f\": \"asdf\", \"g\":"
+            + " null } }");
+    posRow.put(
+        "obj",
+        "{ \"a\": 1, \"b\": \"qwerty\", \"c\": null, \"d\": { \"e\": 2, \"f\": \"asdf\", \"g\":"
+            + " null } }");
+    posRow.put("arr", Arrays.asList("{ \"a\": 1}", "{ \"b\": 2 }", "{ \"c\": 3 }"));
+    posRow.put("epochdays", "2022-09-18 20:05:07"); // DATE, 18.09.2022
+    posRow.put("epochsec", "2022-09-18 20:05:07"); // TIMESTAMP_NTZ(0)
+    posRow.put("epochnano", "2022-09-18 20:05:07.999999999"); // TIMESTAMP_NTZ(9)
+    posRow.put("timesec", "01:00:01.999999999"); // TIME(0)
+    posRow.put("timenano", "01:00:01.999999999"); // TIME(9)
+
+    return posRow;
   }
 
-  private static byte[] nextBytes(Random r) {
-    byte[] bin = new byte[128];
-    r.nextBytes(bin);
-    for (int i = 0; i < bin.length; i++) {
-      bin[i] = (byte) (Math.abs(bin[i]) % 25 + 97); // ascii letters
-    }
-    return bin;
+  private void assertNegRow(Map<String, Object> expectedNegRow, ResultSet actualResult)
+      throws SQLException {
+    assertNonTimeAndVarFields(expectedNegRow, actualResult);
+    Assert.assertEquals("{}", actualResult.getString("VAR"));
+    Assert.assertEquals("{}", actualResult.getString("OBJ"));
+    Assert.assertEquals("[]", actualResult.getString("ARR"));
+    Assert.assertEquals(0, actualResult.getDate("EPOCHDAYS").getTime());
+    Assert.assertEquals(0, actualResult.getTimestamp("EPOCHSEC").getTime());
+    Assert.assertEquals(0, actualResult.getTimestamp("EPOCHNANO").getNanos());
+    Assert.assertEquals(0, actualResult.getTimestamp("EPOCHNANO").getTime());
+    Assert.assertEquals(0, actualResult.getTimestamp("TIMESEC").getTime());
+    Assert.assertEquals(0, actualResult.getTimestamp("TIMENANO").getNanos());
+    Assert.assertEquals(0, actualResult.getTimestamp("TIMENANO").getTime());
+  }
+
+  private void assertNullRow(Map<String, Object> expectedNullRow, ResultSet actualResult)
+      throws SQLException {
+    Assert.assertNotNull(actualResult);
+    assertNonTimeAndVarFields(expectedNullRow, actualResult);
+    Assert.assertEquals(null, actualResult.getString("VAR"));
+    Assert.assertEquals(null, actualResult.getString("OBJ"));
+    Assert.assertEquals(null, actualResult.getString("ARR"));
+    Assert.assertEquals(null, actualResult.getDate("EPOCHDAYS"));
+    Assert.assertEquals(null, actualResult.getTimestamp("EPOCHSEC"));
+    Assert.assertEquals(null, actualResult.getTimestamp("EPOCHNANO"));
+    Assert.assertEquals(null, actualResult.getTimestamp("TIMESEC"));
+    Assert.assertEquals(null, actualResult.getTimestamp("TIMENANO"));
   }
 
-  private static double nextFloat(Random r) {
-    return (r.nextLong() % Math.round(Math.pow(10, 10))) / 100000d;
+  private void assertPosRow(Map<String, Object> expectedPosRow, ResultSet actualResult)
+      throws SQLException {
+    String formattedJSON =
+        "{\n"
+            + "  \"a\": 1,\n"
+            + "  \"b\": \"qwerty\",\n"
+            + "  \"c\": null,\n"
+            + "  \"d\": {\n"
+            + "    \"e\": 2,\n"
+            + "    \"f\": \"asdf\",\n"
+            + "    \"g\": null\n"
+            + "  }\n"
+            + "}";
+
+    String formattedArray =
+        "[\n"
+            + "  \"{ \\\"a\\\": 1}\",\n"
+            + "  \"{ \\\"b\\\": 2 }\",\n"
+            + "  \"{ \\\"c\\\": 3 }\"\n"
+            + "]";
+    Assert.assertNotNull(actualResult);
+    assertNonTimeAndVarFields(expectedPosRow, actualResult);
+    Assert.assertEquals(formattedJSON, actualResult.getString("VAR"));
+    Assert.assertEquals(formattedJSON, actualResult.getString("OBJ"));
+    Assert.assertEquals(formattedArray, actualResult.getString("ARR"));
+    Assert.assertEquals(
+        1663459200000l, actualResult.getDate("EPOCHDAYS").getTime()); // in ms, 18.09.2022 00:00:00
+    Assert.assertEquals(1663531507000L, actualResult.getTimestamp("EPOCHSEC").getTime());
+    Assert.assertEquals(999999999L, actualResult.getTimestamp("EPOCHNANO").getNanos());
+    // 1663531507000 ms + 999 ms (from ns)
+    Assert.assertEquals(1663531507999L, actualResult.getTimestamp("EPOCHNANO").getTime());
+    Assert.assertEquals(3601000, actualResult.getTimestamp("TIMESEC").getTime()); // 1h + 1s in ms
+    Assert.assertEquals(999999999, actualResult.getTimestamp("TIMENANO").getNanos());
+    // 1h + 1s + 999 ms (from ns)
+    Assert.assertEquals(3601999, actualResult.getTimestamp("TIMENANO").getTime());
   }
 
-  private static String nextJson(Random r) {
-    return String.format(
-        "{ \"%s\": %d, \"%s\": \"%s\", \"%s\": null, \"%s\": { \"%s\": %f, \"%s\": \"%s\", \"%s\":"
-            + " null } }",
-        nextString(r),
-        r.nextInt(),
-        nextString(r),
-        nextString(r),
-        nextString(r),
-        nextString(r),
-        nextString(r),
-        r.nextFloat(),
-        nextString(r),
-        nextString(r),
-        nextString(r));
+  private void assertNonTimeAndVarFields(Map<String, Object> row, ResultSet result)
+      throws SQLException {
+    Assert.assertNotNull(result);
+    Assert.assertEquals(row.get("num"), result.getInt("NUM"));
+    Assert.assertEquals(row.get("num_10_5"), result.getBigDecimal("NUM_10_5"));
+    Assert.assertEquals(row.get("num_38_0"), result.getBigDecimal("NUM_38_0"));
+    Assert.assertEquals(row.get("num_2_0"), result.getInt("NUM_2_0"));
+    Assert.assertEquals(row.get("num_4_0"), result.getInt("NUM_4_0"));
+    Assert.assertEquals(row.get("num_9_0"), result.getInt("NUM_9_0"));
+    Assert.assertEquals((long) row.get("num_18_0"), result.getLong("NUM_18_0"));
+    Assert.assertEquals((float) row.get("num_float"), result.getFloat("NUM_FLOAT"), 0);
+    Assert.assertEquals(row.get("str_varchar"), result.getString("STR_VARCHAR"));
+    Assert.assertArrayEquals((byte[]) row.get("bin"), result.getBytes("BIN"));
+    Assert.assertEquals(row.get("bl"), result.getBoolean("BL"));
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index 0ea0e5de4..31afda1f0 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -9,6 +9,8 @@
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
@@ -19,12 +21,27 @@
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.SFException;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
+@RunWith(Parameterized.class)
 public abstract class AbstractDataTypeTest {
+  @Parameterized.Parameters(name = "{0}")
+  public static Collection<Object[]> bdecVersion() {
+    return Arrays.asList(
+        new Object[][] {
+          {"Arrow", Constants.BdecVersion.ONE},
+          // TODO: uncomment once SNOW-659721 is deployed and we set the parameter
+          // DISABLE_PARQUET_CACHE to true for the test account
+          // {"Parquet", Constants.BdecVersion.THREE}
+        });
+  }
+
   private static final String SOURCE_COLUMN_NAME = "source";
   private static final String VALUE_COLUMN_NAME = "value";
 
@@ -52,6 +69,13 @@ protected String randomString() {
 
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
+  private final Constants.BdecVersion bdecVersion;
+
+  public AbstractDataTypeTest(
+      @SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
+    this.bdecVersion = bdecVersion;
+  }
+
   @Before
   public void before() throws Exception {
     databaseName = String.format("SDK_DATATYPE_COMPATIBILITY_IT_%s", randomString());
@@ -65,7 +89,13 @@ public void before() throws Exception {
 
     conn.createStatement().execute(String.format("use warehouse %s;", TestUtils.getWarehouse()));
 
-    Properties props = TestUtils.getProperties();
+    if (bdecVersion == Constants.BdecVersion.THREE) {
+      // TODO: encryption and interleaved mode are not yet supported by server side's Parquet
+      // scanner if local file cache is enabled (SNOW-656500)
+      conn.createStatement().execute("alter session set disable_parquet_cache=true;");
+    }
+
+    Properties props = TestUtils.getProperties(bdecVersion);
     if (props.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       props.setProperty(ROLE, "ACCOUNTADMIN");
     }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
index f0b0d8199..a7b9d6cd7 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
@@ -1,10 +1,15 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Ignore;
 import org.junit.Test;
 
 public class BinaryIT extends AbstractDataTypeTest {
 
+  public BinaryIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testBinarySimple() throws Exception {
     testJdbcTypeCompatibility("BINARY", new byte[0], new ByteArrayProvider());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
index bad5eac16..3a9445ea2 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
@@ -8,6 +8,7 @@
 import java.time.OffsetTime;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Ignore;
 import org.junit.Test;
 
@@ -17,6 +18,10 @@
  */
 public class DateTimeIT extends AbstractDataTypeTest {
 
+  public DateTimeIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testTimestampWithTimeZone() throws Exception {
     useLosAngelesTimeZone();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
index 1fdd2ad5e..48936b950 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
@@ -2,9 +2,15 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class LogicalTypesIT extends AbstractDataTypeTest {
+
+  public LogicalTypesIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testLogicalTypes() throws Exception {
     // Test booleans
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
index 57867427d..22c59d3ea 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
@@ -1,9 +1,15 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
 import java.util.Arrays;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class NullIT extends AbstractDataTypeTest {
+
+  public NullIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testNullIngestion() throws Exception {
     for (String type :
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
index 1b1c8d280..64c9bc3fc 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
@@ -2,9 +2,15 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class NumericTypesIT extends AbstractDataTypeTest {
+
+  public NumericTypesIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testIntegers() throws Exception {
     // test bytes
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
index 3c4ddde22..1195d2bfc 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
@@ -15,6 +15,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -24,6 +25,10 @@ public class SemiStructuredIT extends AbstractDataTypeTest {
   // server-side representation. Validation leaves a small buffer for this difference.
   private static final int MAX_ALLOWED_LENGTH = 16 * 1024 * 1024 - 64;
 
+  public SemiStructuredIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testVariant() throws Exception {
     // Test dates
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
index 4bcf39519..198d1c99c 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
@@ -2,10 +2,16 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Ignore;
 import org.junit.Test;
 
 public class StringsIT extends AbstractDataTypeTest {
+
+  public StringsIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
   @Test
   public void testStrings() throws Exception {
     testJdbcTypeCompatibility("VARCHAR", "", new StringProvider());

From e4d44b4f690e920f2b98586b78ebfa5ffec0ce38 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Tue, 8 Nov 2022 15:52:41 +0100
Subject: [PATCH 086/356] SNOW-637237 Retry for 2 minutes since the first
 failure (#281)

Retry HTTP requests for some fixed total time instead of fixed number
of attempts. This provides better balance between retrying "long enough"
and not running out of memory.

1. If the server fails quickly, it is desirable to retry more times.
2. If the server fails slowly (times out after 1 minute, for example),
   many retry attempts can be cause the SDK to run out of memory
   due to the large amount of buffered data.

Retrying for N minutes makes is easier to reason about the amount
of data that is going to be buffered while waiting for the server
to recover.
---
 .../net/snowflake/ingest/utils/HttpUtil.java  | 37 +++++++++++++++----
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index a8d08849f..602ca3b6c 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -8,6 +8,7 @@
 
 import java.security.Security;
 import java.time.Duration;
+import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.Properties;
 import java.util.Set;
@@ -51,6 +52,8 @@ public class HttpUtil {
   public static final String HTTP_PROXY_PASSWORD = "http.proxyPassword";
 
   private static final String PROXY_SCHEME = "http";
+  private static final String FIRST_FAULT_TIMESTAMP = "FIRST_FAULT_TIMESTAMP";
+  private static final Duration TOTAL_RETRY_DURATION = Duration.of(120, ChronoUnit.SECONDS);
   private static final Duration RETRY_INTERVAL = Duration.of(3, ChronoUnit.SECONDS);
   private static final int MAX_RETRIES = 3;
   private static volatile CloseableHttpClient httpClient;
@@ -203,23 +206,41 @@ private static ServiceUnavailableRetryStrategy getServiceUnavailableRetryStrateg
       @Override
       public boolean retryRequest(
           final HttpResponse response, final int executionCount, final HttpContext context) {
-        int statusCode = response.getStatusLine().getStatusCode();
-        if (executionCount == MAX_RETRIES + 1) {
-          LOGGER.info("Reached the max retry time, not retrying anymore");
-          return false;
+        Object firstFault = context.getAttribute(FIRST_FAULT_TIMESTAMP);
+        long totalRetryDurationSoFarInSeconds = 0;
+        if (firstFault == null) {
+          context.setAttribute(FIRST_FAULT_TIMESTAMP, Instant.now());
+        } else {
+          Instant firstFaultInstant = (Instant) firstFault;
+          Instant now = Instant.now();
+          totalRetryDurationSoFarInSeconds = Duration.between(firstFaultInstant, now).getSeconds();
+
+          if (totalRetryDurationSoFarInSeconds > TOTAL_RETRY_DURATION.getSeconds()) {
+            LOGGER.info(
+                String.format(
+                    "Reached the max retry time of %d seconds, not retrying anymore",
+                    TOTAL_RETRY_DURATION.getSeconds()));
+            return false;
+          }
         }
+
+        int statusCode = response.getStatusLine().getStatusCode();
         boolean needNextRetry =
             (statusCode == REQUEST_TIMEOUT
-                    || statusCode == TOO_MANY_REQUESTS
-                    || statusCode >= SERVER_ERRORS)
-                && executionCount < MAX_RETRIES + 1;
+                || statusCode == TOO_MANY_REQUESTS
+                || statusCode >= SERVER_ERRORS);
         if (needNextRetry) {
           long interval = getRetryInterval();
           LOGGER.warn(
               "In retryRequest for service unavailability with statusCode:{} and uri:{}",
               statusCode,
               getRequestUriFromContext(context));
-          LOGGER.info("Sleep time in millisecond: {}, retryCount: {}", interval, executionCount);
+          LOGGER.info(
+              "Sleep time in millisecond: {}, retryCount: {}, total retry duration: {}s / {}s",
+              interval,
+              executionCount,
+              totalRetryDurationSoFarInSeconds,
+              TOTAL_RETRY_DURATION.getSeconds());
         }
         return needNextRetry;
       }

From b83c9f46e6b5a4c4028dac8e934934dac0d54d6e Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Tue, 8 Nov 2022 12:42:27 +0000
Subject: [PATCH 087/356] Revert "@snow SNOW-612516 Streaming ingest: add
 random file reader to scan single columns, format version 2"

This reverts commit ffe1fb92b22e8bfdbd5af1b16de4a9530988126d (#191).
---
 pom.xml                                       |   5 -
 .../ArrowFileWriterWithCompression.java       |  65 -----------
 .../streaming/internal/ArrowFlusher.java      |  29 +----
 .../streaming/internal/ArrowRowBuffer.java    |   5 +-
 .../internal/CustomCompressionCodec.java      | 108 ------------------
 .../streaming/internal/FlushService.java      |   2 +-
 .../streaming/internal/ParquetRowBuffer.java  |   3 +-
 .../ingest/streaming/internal/RowBuffer.java  |   4 +-
 ...owflakeStreamingIngestChannelInternal.java |   1 -
 .../net/snowflake/ingest/utils/Constants.java |  22 +---
 10 files changed, 9 insertions(+), 235 deletions(-)
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ArrowFileWriterWithCompression.java
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/CustomCompressionCodec.java

diff --git a/pom.xml b/pom.xml
index ee0cd44f6..74419f9e1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -287,11 +287,6 @@
             <version>${arrow.version}</version>
             <scope>runtime</scope>
         </dependency>
-        <dependency>
-            <groupId>org.apache.arrow</groupId>
-            <artifactId>arrow-compression</artifactId>
-            <version>${arrow.version}</version>
-        </dependency>
 
         <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop -->
         <dependency>
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFileWriterWithCompression.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFileWriterWithCompression.java
deleted file mode 100644
index a07509e7c..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFileWriterWithCompression.java
+++ /dev/null
@@ -1,65 +0,0 @@
-package net.snowflake.ingest.streaming.internal;
-
-import java.io.IOException;
-import java.nio.channels.WritableByteChannel;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.VectorUnloader;
-import org.apache.arrow.vector.compression.CompressionCodec;
-import org.apache.arrow.vector.compression.NoCompressionCodec;
-import org.apache.arrow.vector.ipc.ArrowFileWriter;
-import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
-
-/**
- * {@link ArrowFileWriter} with configurable {@link CompressionCodec}
- *
- * <p>{@link ArrowFileWriter} cannot be configured to use a custom {@link CompressionCodec}. {@link
- * ArrowFileWriter} internally has a {@link VectorUnloader} that can be configured with a custom
- * {@link CompressionCodec} but {@link ArrowFileWriter} does not forward this configuration in its
- * public constructors. This class extends {@link ArrowFileWriter} to override the underlying {@link
- * VectorUnloader} and configure it with a custom {@link CompressionCodec}.
- */
-class ArrowFileWriterWithCompression extends ArrowFileWriter {
-  private final VectorUnloader unloader;
-  private final boolean compressed;
-
-  ArrowFileWriterWithCompression(
-      VectorSchemaRoot root, WritableByteChannel out, CompressionCodec codec) {
-    // Note: the dictionary provider is null, this allows to simplify writeBatch() in this class
-    // and exclude ensureDictionariesWritten() call from the original ArrowWriter#writeBatch()
-    // implementation
-    super(root, null, out);
-
-    // we have to create an unloader, different to ArrowWriter#unloader
-    // because it is private and we need to configure it with CompressionCodec
-    // that is not available in ArrowWriter
-    this.unloader = new VectorUnloader(root, true, codec, true);
-    this.compressed = codec != NoCompressionCodec.INSTANCE;
-  }
-
-  // same as ArrowWriter#writeBatch but does not write dictionaries
-  // and uses an unloader configured with a custom CompressionCodec
-  @Override
-  public void writeBatch() throws IOException {
-    start(); // same as ensureStarted() in ArrowWriter#writeBatch()
-
-    // not called as in ArrowWriter#writeBatch
-    // because the dictionary provider is always null in ArrowFileWriterWithCompression()
-    // constructor
-    // ensureDictionariesWritten();
-
-    // ArrowWriter#unloader is private but we need to configure it with CompressionCodec
-    // hence we use here a different unloader
-    // the compression happenes in unloader.getRecordBatch() -> appendNodes() ->
-    // buffers.add(codec.compress(..)..)
-    try (ArrowRecordBatch batch = unloader.getRecordBatch()) {
-      writeRecordBatch(batch);
-      if (compressed) {
-        // compression creates new compressed buffers added to the ArrowRecordBatch
-        // their reference counter is incremented twice:
-        // once in CompressionCodec on create and once in ArrowRecordBatch to retain
-        // first the buffers are released here, second in the ArrowRecordBatch#close on try exit
-        batch.getBuffers().forEach(arrowBuf -> arrowBuf.getReferenceManager().release());
-      }
-    }
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
index 05bfd047d..c87591be2 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
@@ -6,18 +6,15 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.nio.channels.Channels;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.SFException;
 import org.apache.arrow.vector.VectorLoader;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.VectorUnloader;
-import org.apache.arrow.vector.compression.CompressionUtil;
 import org.apache.arrow.vector.ipc.ArrowStreamWriter;
 import org.apache.arrow.vector.ipc.ArrowWriter;
 import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
@@ -29,12 +26,6 @@
 public class ArrowFlusher implements Flusher<VectorSchemaRoot> {
   private static final Logging logger = new Logging(ArrowFlusher.class);
 
-  private final Constants.BdecVersion bdecVersion;
-
-  public ArrowFlusher(Constants.BdecVersion bdecVersion) {
-    this.bdecVersion = bdecVersion;
-  }
-
   @Override
   public Flusher.SerializationResult serialize(
       List<ChannelData<VectorSchemaRoot>> channelsDataPerTable,
@@ -71,13 +62,7 @@ public Flusher.SerializationResult serialize(
         if (root == null) {
           columnEpStatsMapCombined = data.getColumnEps();
           root = data.getVectors();
-          arrowWriter =
-              getArrowBatchWriteMode() == Constants.ArrowBatchWriteMode.STREAM
-                  ? new ArrowStreamWriter(root, null, chunkData)
-                  : new ArrowFileWriterWithCompression(
-                      root,
-                      Channels.newChannel(chunkData),
-                      new CustomCompressionCodec(CompressionUtil.CodecType.ZSTD));
+          arrowWriter = new ArrowStreamWriter(root, null, chunkData);
           loader = new VectorLoader(root);
           firstChannel = data.getChannel();
           arrowWriter.start();
@@ -120,16 +105,4 @@ public Flusher.SerializationResult serialize(
     return new Flusher.SerializationResult(
         channelsMetadataList, columnEpStatsMapCombined, rowCount);
   }
-
-  private Constants.ArrowBatchWriteMode getArrowBatchWriteMode() {
-    switch (bdecVersion) {
-      case ONE:
-        return Constants.ArrowBatchWriteMode.STREAM;
-      case TWO:
-        return Constants.ArrowBatchWriteMode.FILE;
-      default:
-        throw new SFException(
-            ErrorCode.INTERNAL_ERROR, "Unsupported BLOB_FORMAT_VERSION: " + bdecVersion);
-    }
-  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 2d7d17a92..352b27e46 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -16,7 +16,6 @@
 import java.util.Set;
 import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.SFException;
@@ -770,8 +769,8 @@ private int getColumnScale(Map<String, String> metadata) {
   }
 
   @Override
-  public Flusher<VectorSchemaRoot> createFlusher(Constants.BdecVersion bdecVersion) {
-    return new ArrowFlusher(bdecVersion);
+  public Flusher<VectorSchemaRoot> createFlusher() {
+    return new ArrowFlusher();
   }
 
   @VisibleForTesting
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/CustomCompressionCodec.java b/src/main/java/net/snowflake/ingest/streaming/internal/CustomCompressionCodec.java
deleted file mode 100644
index 8c668b114..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/CustomCompressionCodec.java
+++ /dev/null
@@ -1,108 +0,0 @@
-package net.snowflake.ingest.streaming.internal;
-
-import org.apache.arrow.compression.ZstdCompressionCodec;
-import org.apache.arrow.memory.ArrowBuf;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.util.MemoryUtil;
-import org.apache.arrow.vector.compression.CompressionCodec;
-import org.apache.arrow.vector.compression.CompressionUtil;
-
-/**
- * This is custom wrapper on top of {@link CompressionCodec} implementations.
- *
- * <p>The main stream {@link CompressionCodec} implementations are based on {@link
- * org.apache.arrow.vector.compression.AbstractCompressionCodec}. This implementation is similar to
- * it with some tweaks.
- *
- * <p>The reason to have this custom implementation is to fix certain issues with mainstream codecs
- * that lead to buffer reference counting problems due to creation of new compressed buffers and
- * releasing uncompressed. The release of uncompressed buffer from the original vectors to write
- * into file results in double release when the original vector gets released as usual.
- *
- * <p>The custom code extraction is not optimal in a long term and there is a TODO: SNOW-629844 - to
- * follow up on this.
- */
-public class CustomCompressionCodec implements CompressionCodec {
-  private final CompressionCodec actualCodec;
-
-  public CustomCompressionCodec(CompressionUtil.CodecType codecType) {
-    switch (codecType) {
-        // TODO: SNOW-629836 - LZ4 compression currently breaks server side decompression
-      case ZSTD:
-        actualCodec =
-            new ZstdCompressionCodec() {
-              @Override
-              public ArrowBuf compress(BufferAllocator allocator, ArrowBuf uncompressedBuffer) {
-                return doCompress(allocator, uncompressedBuffer);
-              }
-            };
-        break;
-      case NO_COMPRESSION:
-        actualCodec = new CustomNoCompressionCodec();
-        break;
-      default:
-        throw new IllegalArgumentException("unsupported compression type: " + codecType);
-    }
-  }
-
-  @Override
-  public ArrowBuf compress(BufferAllocator allocator, ArrowBuf uncompressedBuffer) {
-    if (uncompressedBuffer.writerIndex() == 0L) {
-      // shortcut for empty buffer
-      ArrowBuf compressedBuffer = allocator.buffer(CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH);
-      compressedBuffer.setLong(0, 0);
-      compressedBuffer.writerIndex(CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH);
-      uncompressedBuffer.close();
-      return compressedBuffer;
-    }
-
-    ArrowBuf compressedBuffer = actualCodec.compress(allocator, uncompressedBuffer);
-    // Here we simplify the original AbstractCompressionCodec#compress
-    // and always write the compressed data even if the compression does not give anything
-    // in case of e.g. random integers.
-    // The reason is that mixing compressed and uncompressed buffers breaks the server side
-    // decompression that happens to expect only compressed buffers for version 8.0.0.
-    long uncompressedLength = uncompressedBuffer.writerIndex();
-    writeUncompressedLength(compressedBuffer, uncompressedLength);
-
-    // Here we again simplify the original AbstractCompressionCodec#compress
-    // and do not release the uncompressedBuffer because it is part of the root vector to write.
-    // This root vector will be released as usual by client.
-    return compressedBuffer;
-  }
-
-  @Override
-  public ArrowBuf decompress(BufferAllocator allocator, ArrowBuf compressedBuffer) {
-    throw new UnsupportedOperationException("decompress is not supported");
-  }
-
-  protected void writeUncompressedLength(ArrowBuf compressedBuffer, long uncompressedLength) {
-    if (!MemoryUtil.LITTLE_ENDIAN) {
-      uncompressedLength = Long.reverseBytes(uncompressedLength);
-    }
-    // first 8 bytes reserved for uncompressed length, according to the specification
-    compressedBuffer.setLong(0, uncompressedLength);
-  }
-
-  @Override
-  public CompressionUtil.CodecType getCodecType() {
-    return actualCodec.getCodecType();
-  }
-
-  private static class CustomNoCompressionCodec implements CompressionCodec {
-    @Override
-    public ArrowBuf compress(BufferAllocator allocator, ArrowBuf uncompressedBuffer) {
-      return CompressionUtil.packageRawBuffer(allocator, uncompressedBuffer);
-    }
-
-    @Override
-    public ArrowBuf decompress(BufferAllocator allocator, ArrowBuf compressedBuffer) {
-      return compressedBuffer;
-    }
-
-    @Override
-    public CompressionUtil.CodecType getCodecType() {
-      return CompressionUtil.CodecType.NO_COMPRESSION;
-    }
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 0ef790c6f..86b586e96 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -449,7 +449,7 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
       SnowflakeStreamingIngestChannelInternal<T> firstChannel =
           channelsDataPerTable.get(0).getChannel();
 
-      Flusher<T> flusher = firstChannel.getRowBuffer().createFlusher(bdecVersion);
+      Flusher<T> flusher = firstChannel.getRowBuffer().createFlusher();
       Flusher.SerializationResult result =
           flusher.serialize(channelsDataPerTable, chunkData, filePath);
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 3f24d4128..552542916 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -16,7 +16,6 @@
 import java.util.Set;
 import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.Pair;
 import org.apache.parquet.column.ColumnDescriptor;
@@ -217,7 +216,7 @@ public void close(String name) {
   }
 
   @Override
-  public Flusher<ParquetChunkData> createFlusher(Constants.BdecVersion bdecVerion) {
+  public Flusher<ParquetChunkData> createFlusher() {
     return new ParquetFlusher(schema);
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
index 2aa9d9861..c32720c42 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
@@ -7,7 +7,6 @@
 import java.util.List;
 import java.util.Map;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
-import net.snowflake.ingest.utils.Constants;
 
 /**
  * Interface for the buffer in the Streaming Ingest channel that holds the un-flushed rows, these
@@ -58,8 +57,7 @@ interface RowBuffer<T> {
    * Create {@link Flusher} implementation to flush the buffered rows to the underlying format
    * implementation for faster processing.
    *
-   * @param bdecVersion version of the BDEC file format to generate
    * @return flusher
    */
-  Flusher<T> createFlusher(Constants.BdecVersion bdecVersion);
+  Flusher<T> createFlusher();
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index d8a8bcf21..5c7774e24 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -152,7 +152,6 @@ private RowBuffer<T> createRowBuffer(Constants.BdecVersion bdecVersion) {
     // can be probably reconsidered
     switch (bdecVersion) {
       case ONE:
-      case TWO:
         //noinspection unchecked
         return (RowBuffer<T>)
             new ArrowRowBuffer((SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>) this);
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index 68602e0a9..6a479a655 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -61,29 +61,13 @@ public enum WriteMode {
     REST_API,
   }
 
-  /** Thw write mode to generate Arrow BDEC file. */
-  public enum ArrowBatchWriteMode {
-    /** Stream format is produced by {@link org.apache.arrow.vector.ipc.ArrowStreamWriter}. */
-    STREAM,
-
-    /**
-     * File format is produced by {@link org.apache.arrow.vector.ipc.ArrowFileWriter}.
-     *
-     * <p>The file format is same as stream format but it adds a footer at the end of the file. The
-     * footer contains metadata for quick random access of certain column data in batches when it is
-     * being read on server side. This way there is no need to download and parse the whole file if
-     * only certain columns are requested.
-     */
-    FILE,
-  }
-
   /** The write mode to generate Arrow BDEC file. */
   public enum BdecVersion {
-    /** Uses Arrow to generate BDEC chunks with {@link ArrowBatchWriteMode#STREAM}. */
+    /** Uses Arrow to generate BDEC chunks. */
     ONE(1),
 
-    /** Uses Arrow to generate BDEC chunks with {@link ArrowBatchWriteMode#FILE}. */
-    TWO(2),
+    // Unused (previously Arrow with per column compression.
+    // TWO(2),
 
     /**
      * Uses Parquet to generate BDEC chunks with {@link

From 1f9623a3cfcf921f42188c8a5cf9a2261e5d0c2b Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 9 Nov 2022 00:16:24 -0800
Subject: [PATCH 088/356] NO-SNOW: enable code coverage check in the SDK (#283)

enable code coverage check in the SDK
---
 .github/workflows/End2EndTest.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/End2EndTest.yml b/.github/workflows/End2EndTest.yml
index 99522be21..5f04e595f 100644
--- a/.github/workflows/End2EndTest.yml
+++ b/.github/workflows/End2EndTest.yml
@@ -32,4 +32,7 @@ jobs:
           WHITESOURCE_API_KEY: ${{ secrets.WHITESOURCE_API_KEY }}
         continue-on-error: false
         run: |
-          ./scripts/run_gh_actions.sh
\ No newline at end of file
+          ./scripts/run_gh_actions.sh
+
+      - name: Code Coverage
+        uses: codecov/codecov-action@v1
\ No newline at end of file

From 82f6569f89f1dd0132f17f70d1128f4968e6412f Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Wed, 9 Nov 2022 10:51:59 +0000
Subject: [PATCH 089/356] @snow no-snow Streaming ingest: always send BDEC
 version as server side supports this

---
 .../streaming/internal/BlobMetadata.java      | 16 ++++++------
 .../internal/BlobMetadataWithBdecVersion.java | 25 -------------------
 2 files changed, 7 insertions(+), 34 deletions(-)
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index f90ba5055..8b0dd52dd 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -49,16 +49,14 @@ List<ChunkMetadata> getChunks() {
     return this.chunks;
   }
 
-  /**
-   * Create {@link BlobMetadata} in case of {@link Constants.BdecVersion#ONE} and {@link
-   * BlobMetadataWithBdecVersion} otherwise to send BDEC version to server side.
-   */
+  @JsonProperty("bdec_version")
+  byte getVersionByte() {
+    return bdecVersion.toByte();
+  }
+
+  /** Create {@link BlobMetadata}. */
   static BlobMetadata createBlobMetadata(
       String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks) {
-    // TODO SNOW-659721: Unify BlobMetadata with BlobMetadataWithBdecVersion once server side
-    // BdecVersion in production
-    return bdecVersion == Constants.BdecVersion.ONE
-        ? new BlobMetadata(path, md5, bdecVersion, chunks)
-        : new BlobMetadataWithBdecVersion(path, md5, bdecVersion, chunks);
+    return new BlobMetadata(path, md5, bdecVersion, chunks);
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
deleted file mode 100644
index 7f9c602e3..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadataWithBdecVersion.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import java.util.List;
-import net.snowflake.ingest.utils.Constants;
-
-/**
- * Same as {@link BlobMetadata} but with {@link Constants.BdecVersion} to send it to server side if
- * the version is greater than {@link Constants.BdecVersion#ONE}.
- */
-class BlobMetadataWithBdecVersion extends BlobMetadata {
-  BlobMetadataWithBdecVersion(
-      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks) {
-    super(path, md5, bdecVersion, chunks);
-  }
-
-  @JsonProperty("bdec_version")
-  byte getVersionByte() {
-    return getVersion().toByte();
-  }
-}

From 1220753611cf716879a9f3030819a013b9b14359 Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Mon, 14 Nov 2022 11:57:15 +0000
Subject: [PATCH 090/356] @snow no-snow Fix more parquet dependencies causing
 vulnerabilities

---
 pom.xml | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/pom.xml b/pom.xml
index 74419f9e1..db5e48afd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -315,6 +315,26 @@
                     <groupId>org.apache.hadoop</groupId>
                     <artifactId>hadoop-hdfs-client</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.avro</groupId>
+                    <artifactId>avro</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.inject.extensions</groupId>
+                    <artifactId>guice-servlet</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.eclipse.jetty.websocket</groupId>
+                    <artifactId>websocket-client</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>io.netty</groupId>
+                    <artifactId>netty</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -338,10 +358,18 @@
                     <groupId>com.sun.jersey</groupId>
                     <artifactId>jersey-server</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-servlet</artifactId>
+                </exclusion>
                 <exclusion>
                     <groupId>javax.servlet</groupId>
                     <artifactId>servlet-api</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>javax.servlet</groupId>
+                    <artifactId>javax.servlet-api</artifactId>
+                </exclusion>
                 <exclusion>
                     <groupId>javax.servlet.jsp</groupId>
                     <artifactId>jsp-api</artifactId>
@@ -362,6 +390,10 @@
                     <groupId>org.mortbay.jetty</groupId>
                     <artifactId>jetty</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.eclipse.jetty</groupId>
+                    <artifactId>jetty-server</artifactId>
+                </exclusion>
                 <exclusion>
                     <groupId>org.slf4j</groupId>
                     <artifactId>slf4j-log4j12</artifactId>
@@ -370,6 +402,10 @@
                     <groupId>org.apache.zookeeper</groupId>
                     <artifactId>zookeeper</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.avro</groupId>
+                    <artifactId>avro</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 

From ae963a43eba4832d7910d5f9c6b9c6ff89b002ff Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Mon, 14 Nov 2022 23:13:08 +0000
Subject: [PATCH 091/356] small fix for the memory logic

---
 .../internal/SnowflakeStreamingIngestChannelInternal.java   | 5 +++--
 .../internal/SnowflakeStreamingIngestChannelTest.java       | 6 ++++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 5c7774e24..0fc9e76f8 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -477,9 +477,10 @@ private boolean hasLowRuntimeMemory(Runtime runtime) {
         maxMemoryLimitInBytes == MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT
             ? runtime.maxMemory()
             : maxMemoryLimitInBytes;
+    long freeMemory = runtime.freeMemory() + (runtime.maxMemory()-runtime.totalMemory());
     boolean hasLowRuntimeMemory =
-        runtime.freeMemory() < LOW_RUNTIME_MEMORY_THRESHOLD_IN_BYTES
-            && runtime.freeMemory() * 100 / maxMemory < insertThrottleThresholdInPercentage;
+            freeMemory < LOW_RUNTIME_MEMORY_THRESHOLD_IN_BYTES
+            && freeMemory * 100 / maxMemory < insertThrottleThresholdInPercentage;
     if (hasLowRuntimeMemory) {
       logger.logWarn(
           "Throttled due to memory pressure, client={}, channel={}.",
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 3a86ef57a..38651fa42 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -513,6 +513,7 @@ public void testInsertRow() {
 
   @Test
   public void testInsertRowThrottling() {
+    long maxMemory = 1000000L;
     SnowflakeStreamingIngestClientInternal<?> client =
         new SnowflakeStreamingIngestClientInternal<>("client");
     SnowflakeStreamingIngestChannelInternal<?> channel =
@@ -530,11 +531,12 @@ public void testInsertRowThrottling() {
             OpenChannelRequest.OnErrorOption.CONTINUE);
 
     Runtime mockedRunTime = Mockito.mock(Runtime.class);
-    Mockito.when(mockedRunTime.maxMemory()).thenReturn(1000000L);
+    Mockito.when(mockedRunTime.maxMemory()).thenReturn(maxMemory);
+    Mockito.when(mockedRunTime.totalMemory()).thenReturn(maxMemory);
     ParameterProvider parameterProvider = new ParameterProvider();
     Mockito.when(mockedRunTime.freeMemory())
         .thenReturn(
-            1000000L * (parameterProvider.getInsertThrottleThresholdInPercentage() - 1) / 100);
+                maxMemory * (parameterProvider.getInsertThrottleThresholdInPercentage() - 1) / 100);
 
     CompletableFuture<Void> future =
         CompletableFuture.runAsync(() -> channel.throttleInsertIfNeeded(mockedRunTime));

From b9dc7db7d7745831052c2c54f7f3dfe1b0a34f4a Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Tue, 15 Nov 2022 00:05:17 +0000
Subject: [PATCH 092/356] fix format

---
 .../internal/SnowflakeStreamingIngestChannelInternal.java     | 4 ++--
 .../internal/SnowflakeStreamingIngestChannelTest.java         | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 0fc9e76f8..07f58d771 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -477,9 +477,9 @@ private boolean hasLowRuntimeMemory(Runtime runtime) {
         maxMemoryLimitInBytes == MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT
             ? runtime.maxMemory()
             : maxMemoryLimitInBytes;
-    long freeMemory = runtime.freeMemory() + (runtime.maxMemory()-runtime.totalMemory());
+    long freeMemory = runtime.freeMemory() + (runtime.maxMemory() - runtime.totalMemory());
     boolean hasLowRuntimeMemory =
-            freeMemory < LOW_RUNTIME_MEMORY_THRESHOLD_IN_BYTES
+        freeMemory < LOW_RUNTIME_MEMORY_THRESHOLD_IN_BYTES
             && freeMemory * 100 / maxMemory < insertThrottleThresholdInPercentage;
     if (hasLowRuntimeMemory) {
       logger.logWarn(
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 38651fa42..63e99cfff 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -536,7 +536,7 @@ public void testInsertRowThrottling() {
     ParameterProvider parameterProvider = new ParameterProvider();
     Mockito.when(mockedRunTime.freeMemory())
         .thenReturn(
-                maxMemory * (parameterProvider.getInsertThrottleThresholdInPercentage() - 1) / 100);
+            maxMemory * (parameterProvider.getInsertThrottleThresholdInPercentage() - 1) / 100);
 
     CompletableFuture<Void> future =
         CompletableFuture.runAsync(() -> channel.throttleInsertIfNeeded(mockedRunTime));

From b7d2249ed9c287954458229ab1ddda8ecb67809c Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Mon, 14 Nov 2022 16:34:14 -0800
Subject: [PATCH 093/356] V1.0.2-beta.6 release (#286)

This release contains a few bug fixes and improvements for Snowpipe Streaming:
- [Improvement] Add parquet file support, this will be our default file format in the future
- [Improvement] Add support to specify a memory limit per client instead of using the system max memory
- [Improvement] Remove a bunch of unused code and setup code coverage support

This release contains a few bug fixes and improvements for both Snowpipe and Snowpipe Streaming:
- [Improvement] Improve the HTTP retry logic
---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index db5e48afd..da16b3ee4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
     <!-- Arifact name and version information -->
     <groupId>net.snowflake</groupId>
     <artifactId>snowflake-ingest-sdk</artifactId>
-    <version>1.0.2-beta.5</version>
+    <version>1.0.2-beta.6</version>
     <packaging>jar</packaging>
     <name>Snowflake Ingest SDK</name>
     <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 07085d582..2b438a8dd 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -127,7 +127,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.0.2-beta.5";
+  public static final String DEFAULT_VERSION = "1.0.2-beta.6";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 7426f9c6dee8ce42df00ef493a9fe64c95d0d45b Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Tue, 29 Nov 2022 10:41:15 +0100
Subject: [PATCH 094/356] SNOW-637237 Test opening several thousand channels in
 parallel (#294)

---
 .../internal/OpenManyChannelsIT.java          | 192 ++++++++++++++++++
 1 file changed, 192 insertions(+)
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/OpenManyChannelsIT.java

diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/OpenManyChannelsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/OpenManyChannelsIT.java
new file mode 100644
index 000000000..006623728
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/OpenManyChannelsIT.java
@@ -0,0 +1,192 @@
+package net.snowflake.ingest.streaming.internal;
+
+import static net.snowflake.ingest.utils.Constants.ROLE;
+
+import java.sql.Connection;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import net.snowflake.ingest.TestUtils;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/** Tries to open several thousand channels into the same table from multiple threads in parallel */
+public class OpenManyChannelsIT {
+  private static final int THREAD_COUNT = 20;
+  private static final int CHANNELS_PER_THREAD = 250;
+  private static final String SCHEMA_NAME = "PUBLIC";
+  private static final String TABLE_NAME = "T1";
+
+  private String databaseName;
+
+  private Connection conn;
+
+  private SnowflakeStreamingIngestClient client;
+
+  @Before
+  public void setUp() throws Exception {
+    databaseName =
+        String.format("SDK_DATATYPE_COMPATIBILITY_IT_%s", RandomStringUtils.randomNumeric(9));
+    conn = TestUtils.getConnection(true);
+    conn.createStatement().execute(String.format("create or replace database %s;", databaseName));
+    conn.createStatement()
+        .execute(
+            String.format(
+                "create or replace table %s.%s.%s (col int)",
+                databaseName, SCHEMA_NAME, TABLE_NAME));
+    Properties props = TestUtils.getProperties(Constants.BdecVersion.ONE);
+    if (props.getProperty(ROLE).equals("DEFAULT_ROLE")) {
+      props.setProperty(ROLE, "ACCOUNTADMIN");
+    }
+    client = SnowflakeStreamingIngestClientFactory.builder("client1").setProperties(props).build();
+  }
+
+  /**
+   * Reopens the same channel from multiple threads, asserts each channel has a unique client
+   * sequencer. At the end it verifies that only the channel with highest client sequencer can
+   * ingest data
+   */
+  @Test
+  public void reopenSameChannel() throws Exception {
+    String channelName = "CHANNEL";
+    List<Thread> threads = new ArrayList<>();
+    List<Exception> exceptions = Collections.synchronizedList(new ArrayList<>());
+    ConcurrentHashMap<Integer, SnowflakeStreamingIngestChannel> clientSequencerToChannelMap =
+        new ConcurrentHashMap<>();
+
+    for (int i = 0; i < THREAD_COUNT; i++) {
+      Thread t =
+          new Thread(
+              () -> {
+                for (int j = 0; j < CHANNELS_PER_THREAD; j++) {
+                  OpenChannelRequest openChannelRequest =
+                      OpenChannelRequest.builder(channelName)
+                          .setDBName(databaseName)
+                          .setSchemaName(SCHEMA_NAME)
+                          .setTableName(TABLE_NAME)
+                          .setOnErrorOption(OpenChannelRequest.OnErrorOption.ABORT)
+                          .build();
+                  try {
+                    SnowflakeStreamingIngestChannel channel =
+                        client.openChannel(openChannelRequest);
+                    Long channelSequencer =
+                        ((SnowflakeStreamingIngestChannelInternal<Void>) channel)
+                            .getChannelSequencer();
+                    clientSequencerToChannelMap.put(channelSequencer.intValue(), channel);
+                  } catch (Exception e) {
+                    exceptions.add(e);
+                    break;
+                  }
+                }
+              });
+      t.start();
+      threads.add(t);
+    }
+
+    for (Thread t : threads) {
+      t.join();
+    }
+
+    if (!exceptions.isEmpty()) {
+      for (Exception e : exceptions) {
+        e.printStackTrace();
+      }
+      Assert.fail(String.format("Exceptions thrown: %d", exceptions.size()));
+    }
+
+    // Verify that each reopened channel received its own sequencer
+    Assert.assertEquals(THREAD_COUNT * CHANNELS_PER_THREAD, clientSequencerToChannelMap.size());
+
+    // Verify that ingestion into to the channel with the highest client sequencer works
+    int highestSequencer =
+        clientSequencerToChannelMap.keySet().stream().max(Comparator.naturalOrder()).get();
+    Assert.assertEquals(THREAD_COUNT * CHANNELS_PER_THREAD - 1, highestSequencer);
+    String offsetToken = UUID.randomUUID().toString();
+    SnowflakeStreamingIngestChannel lastOpenedChannel =
+        clientSequencerToChannelMap.get(highestSequencer);
+    lastOpenedChannel.insertRow(new HashMap<>(), offsetToken);
+    TestUtils.waitForOffset(lastOpenedChannel, offsetToken);
+
+    // Verify that ingestion into all other channels does not work
+    for (Map.Entry<Integer, SnowflakeStreamingIngestChannel> entry :
+        clientSequencerToChannelMap.entrySet()) {
+      if (entry.getKey() != highestSequencer) {
+        try {
+          entry.getValue().insertRow(new HashMap<>(), UUID.randomUUID().toString());
+          Assert.fail("Ingestion into channel with non-latest client sequencer should fail");
+        } catch (SFException e) {
+          // all good, expected exception has been thrown
+        }
+      }
+    }
+  }
+
+  /** Opens many channels in parallel, checks that each has client sequencer 0 */
+  @Test
+  public void testOpenManyDifferentChannels() throws Exception {
+    List<Thread> threads = new ArrayList<>();
+    List<Exception> exceptions = Collections.synchronizedList(new ArrayList<>());
+    for (int i = 0; i < THREAD_COUNT; i++) {
+      final int threadId = i;
+      Thread t =
+          new Thread(
+              () -> {
+                for (int j = 0; j < CHANNELS_PER_THREAD; j++) {
+                  OpenChannelRequest openChannelRequest =
+                      OpenChannelRequest.builder(String.format("CHANNEL-%s-%s", threadId, j))
+                          .setDBName(databaseName)
+                          .setSchemaName(SCHEMA_NAME)
+                          .setTableName(TABLE_NAME)
+                          .setOnErrorOption(OpenChannelRequest.OnErrorOption.ABORT)
+                          .build();
+                  try {
+                    SnowflakeStreamingIngestChannel channel =
+                        client.openChannel(openChannelRequest);
+                    Long channelSequencer =
+                        ((SnowflakeStreamingIngestChannelInternal<Void>) channel)
+                            .getChannelSequencer();
+                    Assert.assertEquals(0L, channelSequencer.longValue());
+                  } catch (Exception e) {
+                    exceptions.add(e);
+                    break;
+                  }
+                }
+              });
+      t.start();
+      threads.add(t);
+    }
+
+    for (Thread t : threads) {
+      t.join();
+    }
+
+    if (!exceptions.isEmpty()) {
+      for (Exception e : exceptions) {
+        e.printStackTrace();
+      }
+      Assert.fail(String.format("Exceptions thrown: %d", exceptions.size()));
+    }
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    conn.createStatement().execute(String.format("drop database %s;", databaseName));
+    client.close();
+    conn.close();
+  }
+}

From 074bebd501c87cf777e45fadbfb90a5146bfe0f1 Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Thu, 10 Nov 2022 19:37:23 +0000
Subject: [PATCH 095/356] @snow no-snow Streaming ingest: factor out channel
 from the buffer

---
 .../streaming/internal/AbstractRowBuffer.java |  86 ++++++++--
 .../streaming/internal/ArrowFlusher.java      |  15 +-
 .../streaming/internal/ArrowRowBuffer.java    |  28 ++--
 .../streaming/internal/BlobBuilder.java       | 125 +++++++++++++-
 .../streaming/internal/ChannelData.java       |  34 +++-
 .../internal/ChannelFlushContext.java         | 116 +++++++++++++
 .../streaming/internal/ChannelMetadata.java   |   6 +-
 .../internal/ChannelRuntimeState.java         |  63 +++++++
 .../streaming/internal/ChunkMetadata.java     |   8 +-
 .../streaming/internal/FlushService.java      | 100 +----------
 .../streaming/internal/ParquetFlusher.java    |  14 +-
 .../streaming/internal/ParquetRowBuffer.java  |  21 ++-
 ...owflakeStreamingIngestChannelInternal.java | 158 ++++++------------
 ...nowflakeStreamingIngestClientInternal.java |   7 +-
 .../streaming/internal/ArrowBufferTest.java   |  10 +-
 .../streaming/internal/FlushServiceTest.java  |  48 +++---
 .../streaming/internal/RowBufferTest.java     |  27 +--
 .../SnowflakeStreamingIngestChannelTest.java  |   4 +-
 .../SnowflakeStreamingIngestClientTest.java   |  38 ++---
 19 files changed, 568 insertions(+), 340 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ChannelFlushContext.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ChannelRuntimeState.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 2800310e5..4a49ab114 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -13,13 +13,16 @@
 import java.util.Set;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Consumer;
 import java.util.stream.Collectors;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.util.VisibleForTesting;
 
 /**
@@ -147,14 +150,35 @@ public int getOrdinal() {
   // Current buffer size
   private volatile float bufferSize;
 
-  // Reference to the Streaming Ingest channel that owns this buffer
-  @VisibleForTesting final SnowflakeStreamingIngestChannelInternal<T> owningChannel;
-
   // Names of non-nullable columns
   private final Set<String> nonNullableFieldNames;
 
-  AbstractRowBuffer(SnowflakeStreamingIngestChannelInternal<T> channel) {
-    this.owningChannel = channel;
+  // buffer's channel fully qualified name with database, schema and table
+  final String channelFullyQualifiedName;
+
+  // metric callback to report size of inserted rows
+  private final Consumer<Float> rowSizeMetric;
+
+  // Allocator used to allocate the buffers
+  final BufferAllocator allocator;
+
+  // State of the owning channel
+  final ChannelRuntimeState channelState;
+
+  // ON_ERROR option for this channel
+  final OpenChannelRequest.OnErrorOption onErrorOption;
+
+  AbstractRowBuffer(
+      OpenChannelRequest.OnErrorOption onErrorOption,
+      BufferAllocator allocator,
+      String fullyQualifiedChannelName,
+      Consumer<Float> rowSizeMetric,
+      ChannelRuntimeState channelRuntimeState) {
+    this.onErrorOption = onErrorOption;
+    this.rowSizeMetric = rowSizeMetric;
+    this.channelState = channelRuntimeState;
+    this.channelFullyQualifiedName = fullyQualifiedChannelName;
+    this.allocator = allocator;
     this.nonNullableFieldNames = new HashSet<>();
     this.flushLock = new ReentrantLock();
     this.rowCount = 0;
@@ -256,7 +280,7 @@ public InsertValidationResponse insertRows(
     InsertValidationResponse response = new InsertValidationResponse();
     this.flushLock.lock();
     try {
-      if (this.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.CONTINUE) {
+      if (onErrorOption == OpenChannelRequest.OnErrorOption.CONTINUE) {
         // Used to map incoming row(nth row) to InsertError(for nth row) in response
         long rowIndex = 0;
         for (Map<String, Object> row : rows) {
@@ -305,8 +329,8 @@ public InsertValidationResponse insertRows(
                     RowBufferStats.getCombinedStats(stats, this.tempStatsMap.get(colName))));
       }
 
-      this.owningChannel.setOffsetToken(offsetToken);
-      this.owningChannel.collectRowSize(rowSize);
+      this.channelState.setOffsetToken(offsetToken);
+      this.rowSizeMetric.accept(rowSize);
     } finally {
       this.tempStatsMap.values().forEach(RowBufferStats::reset);
       clearTempRows();
@@ -324,7 +348,7 @@ public InsertValidationResponse insertRows(
    */
   @Override
   public ChannelData<T> flush() {
-    logger.logDebug("Start get data for channel={}", this.owningChannel.getFullyQualifiedName());
+    logger.logDebug("Start get data for channel={}", channelFullyQualifiedName);
     if (this.rowCount > 0) {
       Optional<T> oldData = Optional.empty();
       int oldRowCount = 0;
@@ -334,8 +358,7 @@ public ChannelData<T> flush() {
       Map<String, RowBufferStats> oldColumnEps = null;
 
       logger.logDebug(
-          "Arrow buffer flush about to take lock on channel={}",
-          this.owningChannel.getFullyQualifiedName());
+          "Arrow buffer flush about to take lock on channel={}", channelFullyQualifiedName);
 
       this.flushLock.lock();
       try {
@@ -344,8 +367,8 @@ public ChannelData<T> flush() {
           oldData = getSnapshot();
           oldRowCount = this.rowCount;
           oldBufferSize = this.bufferSize;
-          oldRowSequencer = this.owningChannel.incrementAndGetRowSequencer();
-          oldOffsetToken = this.owningChannel.getOffsetToken();
+          oldRowSequencer = this.channelState.incrementAndGetRowSequencer();
+          oldOffsetToken = this.channelState.getOffsetToken();
           oldColumnEps = new HashMap<>(this.statsMap);
           // Reset everything in the buffer once we save all the info
           reset();
@@ -356,7 +379,7 @@ public ChannelData<T> flush() {
 
       logger.logDebug(
           "Arrow buffer flush released lock on channel={}, rowCount={}, bufferSize={}",
-          this.owningChannel.getFullyQualifiedName(),
+          channelFullyQualifiedName,
           oldRowCount,
           oldBufferSize);
 
@@ -365,10 +388,10 @@ public ChannelData<T> flush() {
         data.setVectors(oldData.get());
         data.setRowCount(oldRowCount);
         data.setBufferSize(oldBufferSize);
-        data.setChannel(owningChannel);
         data.setRowSequencer(oldRowSequencer);
         data.setOffsetToken(oldOffsetToken);
         data.setColumnEps(oldColumnEps);
+        data.setFlusherFactory(this::createFlusher);
         return data;
       }
     }
@@ -463,4 +486,37 @@ static EpInfo buildEpInfoFromStats(long rowCount, Map<String, RowBufferStats> co
     }
     return epInfo;
   }
+
+  /** Row buffer factory. */
+  static <T> AbstractRowBuffer<T> createRowBuffer(
+      OpenChannelRequest.OnErrorOption onErrorOption,
+      BufferAllocator allocator,
+      Constants.BdecVersion bdecVersion,
+      String fullyQualifiedChannelName,
+      Consumer<Float> rowSizeMetric,
+      ChannelRuntimeState channelRuntimeState) {
+    switch (bdecVersion) {
+      case ONE:
+        //noinspection unchecked
+        return (AbstractRowBuffer<T>)
+            new ArrowRowBuffer(
+                onErrorOption,
+                allocator,
+                fullyQualifiedChannelName,
+                rowSizeMetric,
+                channelRuntimeState);
+      case THREE:
+        //noinspection unchecked
+        return (AbstractRowBuffer<T>)
+            new ParquetRowBuffer(
+                onErrorOption,
+                allocator,
+                fullyQualifiedChannelName,
+                rowSizeMetric,
+                channelRuntimeState);
+      default:
+        throw new SFException(
+            ErrorCode.INTERNAL_ERROR, "Unsupported BDEC format version: " + bdecVersion);
+    }
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
index c87591be2..3eca44154 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
@@ -37,7 +37,7 @@ public Flusher.SerializationResult serialize(
     VectorSchemaRoot root = null;
     ArrowWriter arrowWriter = null;
     VectorLoader loader = null;
-    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> firstChannel = null;
+    String firstChannelFullyQualifiedTableName = null;
     Map<String, RowBufferStats> columnEpStatsMapCombined = null;
 
     try {
@@ -45,7 +45,7 @@ public Flusher.SerializationResult serialize(
         // Create channel metadata
         ChannelMetadata channelMetadata =
             ChannelMetadata.builder()
-                .setOwningChannel(data.getChannel())
+                .setOwningChannelFromContext(data.getChannelContext())
                 .setRowSequencer(data.getRowSequencer())
                 .setOffsetToken(data.getOffsetToken())
                 .build();
@@ -54,7 +54,7 @@ public Flusher.SerializationResult serialize(
 
         logger.logDebug(
             "Start building channel={}, rowCount={}, bufferSize={} in blob={}",
-            data.getChannel().getFullyQualifiedName(),
+            data.getChannelContext().getFullyQualifiedName(),
             data.getRowCount(),
             data.getBufferSize(),
             filePath);
@@ -64,14 +64,15 @@ public Flusher.SerializationResult serialize(
           root = data.getVectors();
           arrowWriter = new ArrowStreamWriter(root, null, chunkData);
           loader = new VectorLoader(root);
-          firstChannel = data.getChannel();
+          firstChannelFullyQualifiedTableName =
+              data.getChannelContext().getFullyQualifiedTableName();
           arrowWriter.start();
         } else {
           // This method assumes that channelsDataPerTable is grouped by table. We double check
           // here and throw an error if the assumption is violated
-          if (!data.getChannel()
+          if (!data.getChannelContext()
               .getFullyQualifiedTableName()
-              .equals(firstChannel.getFullyQualifiedTableName())) {
+              .equals(firstChannelFullyQualifiedTableName)) {
             throw new SFException(ErrorCode.INVALID_DATA_IN_CHUNK);
           }
 
@@ -91,7 +92,7 @@ public Flusher.SerializationResult serialize(
 
         logger.logDebug(
             "Finish building channel={}, rowCount={}, bufferSize={} in blob={}",
-            data.getChannel().getFullyQualifiedName(),
+            data.getChannelContext().getFullyQualifiedName(),
             data.getRowCount(),
             data.getBufferSize(),
             filePath);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 352b27e46..9b6001d43 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -14,6 +14,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.Consumer;
 import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.utils.ErrorCode;
@@ -77,17 +78,14 @@ class ArrowRowBuffer extends AbstractRowBuffer<VectorSchemaRoot> {
   // Map the column name to Arrow column field
   private final Map<String, Field> fields;
 
-  // Allocator used to allocate the buffers
-  private final BufferAllocator allocator;
-
-  /**
-   * Construct a ArrowRowBuffer object
-   *
-   * @param channel client channel
-   */
-  ArrowRowBuffer(SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel) {
-    super(channel);
-    this.allocator = channel.getAllocator();
+  /** Construct a ArrowRowBuffer object. */
+  ArrowRowBuffer(
+      OpenChannelRequest.OnErrorOption onErrorOption,
+      BufferAllocator allocator,
+      String fullyQualifiedChannelName,
+      Consumer<Float> rowSizeMetric,
+      ChannelRuntimeState channelState) {
+    super(onErrorOption, allocator, fullyQualifiedChannelName, rowSizeMetric, channelState);
     this.fields = new HashMap<>();
   }
 
@@ -111,7 +109,7 @@ public void setupSchema(List<ColumnMetadata> columns) {
       vectors.add(vector);
       this.statsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
 
-      if (this.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.ABORT) {
+      if (onErrorOption == OpenChannelRequest.OnErrorOption.ABORT) {
         FieldVector tempVector = field.createVector(this.allocator);
         tempVectors.add(tempVector);
         this.tempStatsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
@@ -138,7 +136,7 @@ public void close(String name) {
     logger.logInfo(
         "Trying to close arrow buffer for channel={} from function={}, allocatedBeforeRelease={},"
             + " allocatedAfterRelease={}",
-        this.owningChannel.getName(),
+        channelFullyQualifiedName,
         name,
         allocatedBeforeRelease,
         allocatedAfterRelease);
@@ -146,12 +144,12 @@ public void close(String name) {
 
     // If the channel is valid but still has leftover data, throw an exception because it should be
     // cleaned up already before calling close
-    if (allocatedBeforeRelease > 0 && this.owningChannel.isValid()) {
+    if (allocatedBeforeRelease > 0 && channelState.isValid()) {
       throw new SFException(
           ErrorCode.INTERNAL_ERROR,
           String.format(
               "Memory leaked=%d by allocator=%s, channel=%s",
-              allocatedBeforeRelease, this.allocator, this.owningChannel.getFullyQualifiedName()));
+              allocatedBeforeRelease, this.allocator, channelFullyQualifiedName));
     }
   }
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
index 6b76dd600..cac9d185a 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
@@ -18,12 +18,20 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.security.InvalidAlgorithmParameterException;
+import java.security.InvalidKeyException;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.zip.CRC32;
 import java.util.zip.GZIPOutputStream;
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
 import javax.xml.bind.DatatypeConverter;
 import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.Cryptor;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.Pair;
 
@@ -48,6 +56,108 @@ class BlobBuilder {
   private static final ObjectMapper MAPPER = new ObjectMapper();
   private static final Logging logger = new Logging(BlobBuilder.class);
 
+  /**
+   * Builds blob.
+   *
+   * @param filePath Path of the destination file in cloud storage
+   * @param blobData All the data for one blob. Assumes that all ChannelData in the inner List
+   *     belongs to the same table. Will error if this is not the case
+   * @param bdecVersion version of blob
+   * @return {@link Blob} data
+   */
+  static <T> Blob constructBlobAndMetadata(
+      String filePath, List<List<ChannelData<T>>> blobData, Constants.BdecVersion bdecVersion)
+      throws IOException, NoSuchPaddingException, NoSuchAlgorithmException,
+          InvalidAlgorithmParameterException, InvalidKeyException, IllegalBlockSizeException,
+          BadPaddingException {
+    List<ChunkMetadata> chunksMetadataList = new ArrayList<>();
+    List<byte[]> chunksDataList = new ArrayList<>();
+    long curDataSize = 0L;
+    CRC32 crc = new CRC32();
+
+    // TODO: channels with different schema can't be combined even if they belongs to same table
+    for (List<ChannelData<T>> channelsDataPerTable : blobData) {
+      ByteArrayOutputStream chunkData = new ByteArrayOutputStream();
+      ChannelFlushContext firstChannelFlushContext =
+          channelsDataPerTable.get(0).getChannelContext();
+
+      Flusher<T> flusher = channelsDataPerTable.get(0).createFlusher();
+      Flusher.SerializationResult result =
+          flusher.serialize(channelsDataPerTable, chunkData, filePath);
+
+      if (!result.channelsMetadataList.isEmpty()) {
+        Pair<byte[], Integer> compressionResult =
+            compressIfNeededAndPadChunk(
+                filePath,
+                chunkData,
+                Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES,
+                bdecVersion == Constants.BdecVersion.ONE);
+        byte[] compressedAndPaddedChunkData = compressionResult.getFirst();
+        int compressedChunkLength = compressionResult.getSecond();
+
+        // Encrypt the compressed chunk data, the encryption key is derived using the key from
+        // server with the full blob path.
+        // We need to maintain IV as a block counter for the whole file, even interleaved,
+        // to align with decryption on the Snowflake query path.
+        // TODO: address alignment for the header SNOW-557866
+        long iv = curDataSize / Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES;
+        byte[] encryptedCompressedChunkData =
+            Cryptor.encrypt(
+                compressedAndPaddedChunkData,
+                firstChannelFlushContext.getEncryptionKey(),
+                filePath,
+                iv);
+
+        // Compute the md5 of the chunk data
+        String md5 = computeMD5(encryptedCompressedChunkData, compressedChunkLength);
+        int encryptedCompressedChunkDataSize = encryptedCompressedChunkData.length;
+
+        // Create chunk metadata
+        long startOffset = curDataSize;
+        ChunkMetadata chunkMetadata =
+            ChunkMetadata.builder()
+                .setOwningTableFromChannelContext(firstChannelFlushContext)
+                // The start offset will be updated later in BlobBuilder#build to include the blob
+                // header
+                .setChunkStartOffset(startOffset)
+                // The compressedChunkLength is used because it is the actual data size used for
+                // decompression and md5 calculation on server side.
+                .setChunkLength(compressedChunkLength)
+                .setChannelList(result.channelsMetadataList)
+                .setChunkMD5(md5)
+                .setEncryptionKeyId(firstChannelFlushContext.getEncryptionKeyId())
+                .setEpInfo(
+                    AbstractRowBuffer.buildEpInfoFromStats(
+                        result.rowCount, result.columnEpStatsMapCombined))
+                .build();
+
+        // Add chunk metadata and data to the list
+        chunksMetadataList.add(chunkMetadata);
+        chunksDataList.add(encryptedCompressedChunkData);
+        curDataSize += encryptedCompressedChunkDataSize;
+        crc.update(encryptedCompressedChunkData, 0, encryptedCompressedChunkDataSize);
+
+        logger.logInfo(
+            "Finish building chunk in blob={}, table={}, rowCount={}, startOffset={},"
+                + " uncompressedSize={}, compressedChunkLength={}, encryptedCompressedSize={},"
+                + " bdecVersion={}",
+            filePath,
+            firstChannelFlushContext.getFullyQualifiedTableName(),
+            result.rowCount,
+            startOffset,
+            chunkData.size(),
+            compressedChunkLength,
+            encryptedCompressedChunkDataSize,
+            bdecVersion);
+      }
+    }
+
+    // Build blob file bytes
+    byte[] blobBytes =
+        buildBlob(chunksMetadataList, chunksDataList, crc.getValue(), curDataSize, bdecVersion);
+    return new Blob(blobBytes, chunksMetadataList);
+  }
+
   /**
    * Gzip compress the given chunk data
    *
@@ -129,7 +239,7 @@ static Pair<byte[], Integer> compressIfNeededAndPadChunk(
   }
 
   /**
-   * Build the blob file
+   * Build the blob file bytes
    *
    * @param chunksMetadataList List of chunk metadata
    * @param chunksDataList List of chunk data
@@ -139,7 +249,7 @@ static Pair<byte[], Integer> compressIfNeededAndPadChunk(
    * @return The blob file as a byte array
    * @throws JsonProcessingException
    */
-  static byte[] build(
+  static byte[] buildBlob(
       List<ChunkMetadata> chunksMetadataList,
       List<byte[]> chunksDataList,
       long chunksChecksum,
@@ -209,4 +319,15 @@ static String computeMD5(byte[] data, int length) throws NoSuchAlgorithmExceptio
     byte[] digest = md.digest();
     return DatatypeConverter.printHexBinary(digest).toLowerCase();
   }
+
+  /** Blob data to store in a file and register by server side. */
+  static class Blob {
+    final byte[] blobBytes;
+    final List<ChunkMetadata> chunksMetadataList;
+
+    Blob(byte[] blobBytes, List<ChunkMetadata> chunksMetadataList) {
+      this.blobBytes = blobBytes;
+      this.chunksMetadataList = chunksMetadataList;
+    }
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
index a9bb3cf2f..234d72ca9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
@@ -6,6 +6,7 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.function.Supplier;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 
@@ -21,8 +22,9 @@ class ChannelData<T> {
   private T vectors;
   private float bufferSize;
   private int rowCount;
-  private SnowflakeStreamingIngestChannelInternal<T> channel;
   private Map<String, RowBufferStats> columnEps;
+  private ChannelFlushContext channelFlushContext;
+  private Supplier<Flusher<T>> flusherFactory;
 
   // TODO performance test this vs in place update
   /**
@@ -104,16 +106,36 @@ void setBufferSize(float bufferSize) {
     this.bufferSize = bufferSize;
   }
 
-  SnowflakeStreamingIngestChannelInternal<T> getChannel() {
-    return this.channel;
+  public ChannelFlushContext getChannelContext() {
+    return channelFlushContext;
   }
 
-  void setChannel(SnowflakeStreamingIngestChannelInternal<T> channel) {
-    this.channel = channel;
+  public void setChannelContext(ChannelFlushContext channelFlushContext) {
+    this.channelFlushContext = channelFlushContext;
+  }
+
+  public Flusher<T> createFlusher() {
+    return flusherFactory.get();
+  }
+
+  public void setFlusherFactory(Supplier<Flusher<T>> flusherFactory) {
+    this.flusherFactory = flusherFactory;
   }
 
   @Override
   public String toString() {
-    return this.channel.toString();
+    return "ChannelData{"
+        + "rowSequencer="
+        + rowSequencer
+        + ", offsetToken='"
+        + offsetToken
+        + '\''
+        + ", bufferSize="
+        + bufferSize
+        + ", rowCount="
+        + rowCount
+        + ", channelContext="
+        + channelFlushContext
+        + '}';
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelFlushContext.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelFlushContext.java
new file mode 100644
index 000000000..3e5265719
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelFlushContext.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+/**
+ * Channel immutable identification and encryption attributes.
+ *
+ * <p>It is shared with the {@link ChannelData} and used for BDEC flush as well.
+ */
+class ChannelFlushContext {
+  private final String name;
+  private final String fullyQualifiedName;
+  private final String dbName;
+  private final String schemaName;
+  private final String tableName;
+  private final String fullyQualifiedTableName;
+
+  // Sequencer for this channel, corresponding to client sequencer at server side because each
+  // connection to a channel at server side will be seen as a connection from a new client
+  private final Long channelSequencer;
+
+  // Data encryption key
+  private final String encryptionKey;
+
+  // Data encryption key id
+  private final Long encryptionKeyId;
+
+  ChannelFlushContext(
+      String name,
+      String dbName,
+      String schemaName,
+      String tableName,
+      Long channelSequencer,
+      String encryptionKey,
+      Long encryptionKeyId) {
+    this.name = name;
+    this.fullyQualifiedName = String.format("%s.%s.%s.%s", dbName, schemaName, tableName, name);
+    this.dbName = dbName;
+    this.schemaName = schemaName;
+    this.tableName = tableName;
+    this.fullyQualifiedTableName =
+        String.format("%s.%s.%s", this.getDbName(), this.getSchemaName(), this.getTableName());
+    this.channelSequencer = channelSequencer;
+    this.encryptionKey = encryptionKey;
+    this.encryptionKeyId = encryptionKeyId;
+  }
+
+  @Override
+  public String toString() {
+    return "ChannelContext{"
+        + "name='"
+        + getName()
+        + '\''
+        + ", fullyQualifiedName='"
+        + getFullyQualifiedName()
+        + '\''
+        + ", dbName='"
+        + getDbName()
+        + '\''
+        + ", schemaName='"
+        + getSchemaName()
+        + '\''
+        + ", tableName='"
+        + getTableName()
+        + '\''
+        + ", fullyQualifiedTableName='"
+        + getFullyQualifiedTableName()
+        + '\''
+        + ", channelSequencer="
+        + getChannelSequencer()
+        + ", encryptionKey='"
+        + getEncryptionKey()
+        + '\''
+        + ", encryptionKeyId="
+        + getEncryptionKeyId()
+        + '}';
+  }
+
+  String getName() {
+    return name;
+  }
+
+  String getFullyQualifiedName() {
+    return fullyQualifiedName;
+  }
+
+  String getDbName() {
+    return dbName;
+  }
+
+  String getSchemaName() {
+    return schemaName;
+  }
+
+  String getTableName() {
+    return tableName;
+  }
+
+  String getFullyQualifiedTableName() {
+    return fullyQualifiedTableName;
+  }
+
+  Long getChannelSequencer() {
+    return channelSequencer;
+  }
+
+  String getEncryptionKey() {
+    return encryptionKey;
+  }
+
+  Long getEncryptionKeyId() {
+    return encryptionKeyId;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelMetadata.java
index 6d0479f60..6edb29e62 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelMetadata.java
@@ -29,9 +29,9 @@ static class Builder {
     private Long rowSequencer;
     @Nullable private String offsetToken; // offset token could be null
 
-    Builder setOwningChannel(SnowflakeStreamingIngestChannelInternal<?> channel) {
-      this.channelName = channel.getName();
-      this.clientSequencer = channel.getChannelSequencer();
+    Builder setOwningChannelFromContext(ChannelFlushContext channelFlushContext) {
+      this.channelName = channelFlushContext.getName();
+      this.clientSequencer = channelFlushContext.getChannelSequencer();
       return this;
     }
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelRuntimeState.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelRuntimeState.java
new file mode 100644
index 000000000..3d3e5ac1f
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelRuntimeState.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/** Internal state of channel that is used and mutated by both channel and its buffer. */
+class ChannelRuntimeState {
+  // Indicates whether the channel is still valid
+  private volatile boolean isValid;
+
+  // the channel's current offset token
+  private volatile String offsetToken;
+
+  // the channel's current row sequencer
+  private final AtomicLong rowSequencer;
+
+  ChannelRuntimeState(String offsetToken, long rowSequencer, boolean isValid) {
+    this.offsetToken = offsetToken;
+    this.rowSequencer = new AtomicLong(rowSequencer);
+    this.isValid = isValid;
+  }
+
+  /**
+   * Returns whether the channel is in a valid state.
+   *
+   * @return whether the channel is valid
+   */
+  boolean isValid() {
+    return isValid;
+  }
+
+  /** Invalidate the channel that has this state object. */
+  void invalidate() {
+    isValid = false;
+  }
+
+  /** @return current offset token */
+  String getOffsetToken() {
+    return offsetToken;
+  }
+
+  /** @return current offset token after first incrementing it by one. */
+  long incrementAndGetRowSequencer() {
+    return rowSequencer.incrementAndGet();
+  }
+
+  /** @return row sequencer of the channel. */
+  long getRowSequencer() {
+    return rowSequencer.get();
+  }
+
+  /**
+   * Updates the channel's offset token.
+   *
+   * @param offsetToken new offset token
+   */
+  void setOffsetToken(String offsetToken) {
+    this.offsetToken = offsetToken;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChunkMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChunkMetadata.java
index a7626c4fd..2257026cd 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChunkMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChunkMetadata.java
@@ -36,10 +36,10 @@ static class Builder {
     private EpInfo epInfo;
     private Long encryptionKeyId;
 
-    Builder setOwningTable(SnowflakeStreamingIngestChannelInternal channel) {
-      this.dbName = channel.getDBName();
-      this.schemaName = channel.getSchemaName();
-      this.tableName = channel.getTableName();
+    Builder setOwningTableFromChannelContext(ChannelFlushContext channelFlushContext) {
+      this.dbName = channelFlushContext.getDbName();
+      this.schemaName = channelFlushContext.getSchemaName();
+      this.tableName = channelFlushContext.getTableName();
       return this;
     }
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 86b586e96..a6a80f275 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -12,7 +12,6 @@
 import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
 import com.codahale.metrics.Timer;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.security.InvalidAlgorithmParameterException;
@@ -34,14 +33,12 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.zip.CRC32;
 import javax.crypto.BadPaddingException;
 import javax.crypto.IllegalBlockSizeException;
 import javax.crypto.NoSuchPaddingException;
 import net.snowflake.client.jdbc.SnowflakeSQLException;
 import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
 import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.Cryptor;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.Pair;
@@ -437,95 +434,12 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
       throws IOException, NoSuchAlgorithmException, InvalidAlgorithmParameterException,
           NoSuchPaddingException, IllegalBlockSizeException, BadPaddingException,
           InvalidKeyException {
-    List<ChunkMetadata> chunksMetadataList = new ArrayList<>();
-    List<byte[]> chunksDataList = new ArrayList<>();
-    long curDataSize = 0L;
-    CRC32 crc = new CRC32();
     Timer.Context buildContext = Utils.createTimerContext(this.owningClient.buildLatency);
-
-    // TODO: channels with different schema can't be combined even if they belongs to same table
-    for (List<ChannelData<T>> channelsDataPerTable : blobData) {
-      ByteArrayOutputStream chunkData = new ByteArrayOutputStream();
-      SnowflakeStreamingIngestChannelInternal<T> firstChannel =
-          channelsDataPerTable.get(0).getChannel();
-
-      Flusher<T> flusher = firstChannel.getRowBuffer().createFlusher();
-      Flusher.SerializationResult result =
-          flusher.serialize(channelsDataPerTable, chunkData, filePath);
-
-      if (!result.channelsMetadataList.isEmpty()) {
-        Pair<byte[], Integer> compressionResult =
-            BlobBuilder.compressIfNeededAndPadChunk(
-                filePath,
-                chunkData,
-                Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES,
-                bdecVersion == Constants.BdecVersion.ONE);
-        byte[] compressedAndPaddedChunkData = compressionResult.getFirst();
-        int compressedChunkLength = compressionResult.getSecond();
-
-        // Encrypt the compressed chunk data, the encryption key is derived using the key from
-        // server with the full blob path.
-        // We need to maintain IV as a block counter for the whole file, even interleaved,
-        // to align with decryption on the Snowflake query path.
-        // TODO: address alignment for the header SNOW-557866
-        long iv = curDataSize / Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES;
-        byte[] encryptedCompressedChunkData =
-            Cryptor.encrypt(
-                compressedAndPaddedChunkData, firstChannel.getEncryptionKey(), filePath, iv);
-
-        // Compute the md5 of the chunk data
-        String md5 = BlobBuilder.computeMD5(encryptedCompressedChunkData, compressedChunkLength);
-        int encryptedCompressedChunkDataSize = encryptedCompressedChunkData.length;
-
-        // Create chunk metadata
-        long startOffset = curDataSize;
-        ChunkMetadata chunkMetadata =
-            ChunkMetadata.builder()
-                .setOwningTable(firstChannel)
-                // The start offset will be updated later in BlobBuilder#build to include the blob
-                // header
-                .setChunkStartOffset(startOffset)
-                // The compressedChunkLength is used because it is the actual data size used for
-                // decompression and md5 calculation on server side.
-                .setChunkLength(compressedChunkLength)
-                .setChannelList(result.channelsMetadataList)
-                .setChunkMD5(md5)
-                .setEncryptionKeyId(firstChannel.getEncryptionKeyId())
-                .setEpInfo(
-                    AbstractRowBuffer.buildEpInfoFromStats(
-                        result.rowCount, result.columnEpStatsMapCombined))
-                .build();
-
-        // Add chunk metadata and data to the list
-        chunksMetadataList.add(chunkMetadata);
-        chunksDataList.add(encryptedCompressedChunkData);
-        curDataSize += encryptedCompressedChunkDataSize;
-        crc.update(encryptedCompressedChunkData, 0, encryptedCompressedChunkDataSize);
-
-        logger.logInfo(
-            "Finish building chunk in blob={}, table={}, rowCount={}, startOffset={},"
-                + " uncompressedSize={}, compressedChunkLength={}, encryptedCompressedSize={},"
-                + " bdecVersion={}",
-            filePath,
-            firstChannel.getFullyQualifiedTableName(),
-            result.rowCount,
-            startOffset,
-            chunkData.size(),
-            compressedChunkLength,
-            encryptedCompressedChunkDataSize,
-            bdecVersion);
-      }
-    }
-
-    // Build blob file, and then upload to streaming ingest dedicated stage
-    byte[] blob =
-        BlobBuilder.build(
-            chunksMetadataList, chunksDataList, crc.getValue(), curDataSize, bdecVersion);
+    BlobBuilder.Blob blob = BlobBuilder.constructBlobAndMetadata(filePath, blobData, bdecVersion);
     if (buildContext != null) {
       buildContext.stop();
     }
-
-    return upload(filePath, blob, chunksMetadataList);
+    return upload(filePath, blob.blobBytes, blob.chunksMetadataList);
   }
 
   /**
@@ -647,11 +561,11 @@ <CD> void invalidateAllChannelsInBlob(List<List<ChannelData<CD>>> blobData) {
                   this.owningClient
                       .getChannelCache()
                       .invalidateChannelIfSequencersMatch(
-                          channelData.getChannel().getDBName(),
-                          channelData.getChannel().getSchemaName(),
-                          channelData.getChannel().getTableName(),
-                          channelData.getChannel().getName(),
-                          channelData.getChannel().getChannelSequencer());
+                          channelData.getChannelContext().getDbName(),
+                          channelData.getChannelContext().getSchemaName(),
+                          channelData.getChannelContext().getTableName(),
+                          channelData.getChannelContext().getName(),
+                          channelData.getChannelContext().getChannelSequencer());
                 }));
   }
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
index 3d9827b6b..3ea1ed44b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
@@ -53,7 +53,7 @@ public SerializationResult serialize(
       throws IOException {
     List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
     long rowCount = 0L;
-    SnowflakeStreamingIngestChannelInternal<ParquetChunkData> firstChannel = null;
+    String firstChannelFullyQualifiedTableName = null;
     Map<String, RowBufferStats> columnEpStatsMapCombined = null;
     List<List<Object>> rows = null;
 
@@ -61,7 +61,7 @@ public SerializationResult serialize(
       // Create channel metadata
       ChannelMetadata channelMetadata =
           ChannelMetadata.builder()
-              .setOwningChannel(data.getChannel())
+              .setOwningChannelFromContext(data.getChannelContext())
               .setRowSequencer(data.getRowSequencer())
               .setOffsetToken(data.getOffsetToken())
               .build();
@@ -70,7 +70,7 @@ public SerializationResult serialize(
 
       logger.logDebug(
           "Parquet Flusher: Start building channel={}, rowCount={}, bufferSize={} in blob={}",
-          data.getChannel().getFullyQualifiedName(),
+          data.getChannelContext().getFullyQualifiedName(),
           data.getRowCount(),
           data.getBufferSize(),
           filePath);
@@ -78,13 +78,13 @@ public SerializationResult serialize(
       if (rows == null) {
         columnEpStatsMapCombined = data.getColumnEps();
         rows = new ArrayList<>();
-        firstChannel = data.getChannel();
+        firstChannelFullyQualifiedTableName = data.getChannelContext().getFullyQualifiedTableName();
       } else {
         // This method assumes that channelsDataPerTable is grouped by table. We double check
         // here and throw an error if the assumption is violated
-        if (!data.getChannel()
+        if (!data.getChannelContext()
             .getFullyQualifiedTableName()
-            .equals(firstChannel.getFullyQualifiedTableName())) {
+            .equals(firstChannelFullyQualifiedTableName)) {
           throw new SFException(ErrorCode.INVALID_DATA_IN_CHUNK);
         }
 
@@ -97,7 +97,7 @@ public SerializationResult serialize(
 
       logger.logDebug(
           "Parquet Flusher: Finish building channel={}, rowCount={}, bufferSize={} in blob={}",
-          data.getChannel().getFullyQualifiedName(),
+          data.getChannelContext().getFullyQualifiedName(),
           data.getRowCount(),
           data.getBufferSize(),
           filePath);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 552542916..1ef2e854b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -14,10 +14,12 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.Consumer;
 import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.Pair;
+import org.apache.arrow.memory.BufferAllocator;
 import org.apache.parquet.column.ColumnDescriptor;
 import org.apache.parquet.schema.MessageType;
 import org.apache.parquet.schema.PrimitiveType;
@@ -39,13 +41,14 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
 
   private MessageType schema;
 
-  /**
-   * Construct a ParquetRowBuffer object
-   *
-   * @param channel client channel
-   */
-  ParquetRowBuffer(SnowflakeStreamingIngestChannelInternal<ParquetChunkData> channel) {
-    super(channel);
+  /** Construct a ParquetRowBuffer object. */
+  ParquetRowBuffer(
+      OpenChannelRequest.OnErrorOption onErrorOption,
+      BufferAllocator allocator,
+      String fullyQualifiedChannelName,
+      Consumer<Float> rowSizeMetric,
+      ChannelRuntimeState channelRuntimeState) {
+    super(onErrorOption, allocator, fullyQualifiedChannelName, rowSizeMetric, channelRuntimeState);
     fieldIndex = new HashMap<>();
     metadata = new HashMap<>();
     data = new ArrayList<>();
@@ -73,7 +76,7 @@ public void setupSchema(List<ColumnMetadata> columns) {
       }
       this.statsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
 
-      if (this.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.ABORT) {
+      if (onErrorOption == OpenChannelRequest.OnErrorOption.ABORT) {
         this.tempStatsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
       }
 
@@ -211,7 +214,7 @@ public void close(String name) {
     this.fieldIndex.clear();
     logger.logInfo(
         "Trying to close parquet buffer for channel={} from function={}",
-        this.owningChannel.getName(),
+        channelFullyQualifiedName,
         name);
   }
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 07f58d771..3a38f5e7b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -10,11 +10,11 @@
 import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
 import static net.snowflake.ingest.utils.ParameterProvider.MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
@@ -26,7 +26,6 @@
 import net.snowflake.ingest.utils.Utils;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
 
 /**
  * The first version of implementation for SnowflakeStreamingIngestChannel
@@ -38,40 +37,20 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
 
   private static final Logging logger = new Logging(SnowflakeStreamingIngestChannelInternal.class);
 
-  private final String channelName;
-  private final String dbName;
-  private final String schemaName;
-  private final String tableName;
-  private volatile String offsetToken;
-  private final AtomicLong rowSequencer;
-
-  // Sequencer for this channel, corresponding to client sequencer at server side because each
-  // connection to a channel at server side will be seen as a connection from a new client
-  private final Long channelSequencer;
+  // this context contains channel immutable identification and encryption attributes
+  private final ChannelFlushContext channelFlushContext;
 
   // Reference to the row buffer
   private final RowBuffer<T> rowBuffer;
 
-  // Indicates whether the channel is still valid
-  private volatile boolean isValid;
-
   // Indicates whether the channel is closed
   private volatile boolean isClosed;
 
   // Reference to the client that owns this channel
   private final SnowflakeStreamingIngestClientInternal<T> owningClient;
 
-  // Memory allocator
-  private final BufferAllocator allocator;
-
-  // Data encryption key
-  private final String encryptionKey;
-
-  // Data encryption key id
-  private final Long encryptionKeyId;
-
-  // ON_ERROR option for this channel
-  private final OpenChannelRequest.OnErrorOption onErrorOption;
+  // state of the channel that will be shared with its underlying buffer
+  private final ChannelRuntimeState channelState;
 
   /**
    * Constructor for TESTING ONLY which allows us to set the test mode
@@ -128,41 +107,24 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
       OpenChannelRequest.OnErrorOption onErrorOption,
       Constants.BdecVersion bdecVersion,
       BufferAllocator allocator) {
-    this.channelName = name;
-    this.dbName = dbName;
-    this.schemaName = schemaName;
-    this.tableName = tableName;
-    this.offsetToken = offsetToken;
-    this.channelSequencer = channelSequencer;
-    this.rowSequencer = new AtomicLong(rowSequencer);
-    this.isValid = true;
     this.isClosed = false;
     this.owningClient = client;
-    this.allocator = allocator;
-    this.rowBuffer = createRowBuffer(bdecVersion);
-    this.encryptionKey = encryptionKey;
-    this.encryptionKeyId = encryptionKeyId;
-    this.onErrorOption = onErrorOption;
-    logger.logInfo("Channel={} created for table={}", this.channelName, this.tableName);
-  }
-
-  private RowBuffer<T> createRowBuffer(Constants.BdecVersion bdecVersion) {
-    // TODO: The circular dependency SnowflakeStreamingIngestChannelInternal <-> RowBuffer
-    // (SNOW-657667)
-    // can be probably reconsidered
-    switch (bdecVersion) {
-      case ONE:
-        //noinspection unchecked
-        return (RowBuffer<T>)
-            new ArrowRowBuffer((SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot>) this);
-      case THREE:
-        //noinspection unchecked
-        return (RowBuffer<T>)
-            new ParquetRowBuffer((SnowflakeStreamingIngestChannelInternal<ParquetChunkData>) this);
-      default:
-        throw new SFException(
-            ErrorCode.INTERNAL_ERROR, "Unsupported BDEC format version: " + bdecVersion);
-    }
+    this.channelFlushContext =
+        new ChannelFlushContext(
+            name, dbName, schemaName, tableName, channelSequencer, encryptionKey, encryptionKeyId);
+    this.channelState = new ChannelRuntimeState(offsetToken, rowSequencer, true);
+    this.rowBuffer =
+        AbstractRowBuffer.createRowBuffer(
+            onErrorOption,
+            allocator,
+            bdecVersion,
+            getFullyQualifiedName(),
+            this::collectRowSize,
+            channelState);
+    logger.logInfo(
+        "Channel={} created for table={}",
+        this.channelFlushContext.getName(),
+        this.channelFlushContext.getTableName());
   }
 
   /**
@@ -173,8 +135,7 @@ private RowBuffer<T> createRowBuffer(Constants.BdecVersion bdecVersion) {
    */
   @Override
   public String getFullyQualifiedName() {
-    return String.format(
-        "%s.%s.%s.%s", this.dbName, this.schemaName, this.tableName, this.channelName);
+    return channelFlushContext.getFullyQualifiedName();
   }
 
   /**
@@ -184,50 +145,32 @@ public String getFullyQualifiedName() {
    */
   @Override
   public String getName() {
-    return this.channelName;
+    return this.channelFlushContext.getName();
   }
 
   @Override
   public String getDBName() {
-    return this.dbName;
+    return this.channelFlushContext.getDbName();
   }
 
   @Override
   public String getSchemaName() {
-    return this.schemaName;
+    return this.channelFlushContext.getSchemaName();
   }
 
   @Override
   public String getTableName() {
-    return this.tableName;
-  }
-
-  String getOffsetToken() {
-    return this.offsetToken;
-  }
-
-  void setOffsetToken(String offsetToken) {
-    this.offsetToken = offsetToken;
+    return this.channelFlushContext.getTableName();
   }
 
   Long getChannelSequencer() {
-    return this.channelSequencer;
-  }
-
-  long incrementAndGetRowSequencer() {
-    return this.rowSequencer.incrementAndGet();
-  }
-
-  long getRowSequencer() {
-    return this.rowSequencer.get();
-  }
-
-  String getEncryptionKey() {
-    return this.encryptionKey;
+    return this.channelFlushContext.getChannelSequencer();
   }
 
-  Long getEncryptionKeyId() {
-    return this.encryptionKeyId;
+  /** @return current state of the channel */
+  @VisibleForTesting
+  ChannelRuntimeState getChannelState() {
+    return this.channelState;
   }
 
   /**
@@ -237,7 +180,7 @@ Long getEncryptionKeyId() {
    */
   @Override
   public String getFullyQualifiedTableName() {
-    return String.format("%s.%s.%s", this.dbName, this.schemaName, this.tableName);
+    return channelFlushContext.getFullyQualifiedTableName();
   }
 
   /**
@@ -246,24 +189,28 @@ public String getFullyQualifiedTableName() {
    * @return a ChannelData object
    */
   ChannelData<T> getData() {
-    return this.rowBuffer.flush();
+    ChannelData<T> data = this.rowBuffer.flush();
+    if (data != null) {
+      data.setChannelContext(channelFlushContext);
+    }
+    return data;
   }
 
   /** @return a boolean to indicate whether the channel is valid or not */
   @Override
   public boolean isValid() {
-    return this.isValid;
+    return this.channelState.isValid();
   }
 
   /** Mark the channel as invalid, and release resources */
   void invalidate(String message) {
-    this.isValid = false;
+    this.channelState.invalidate();
     this.rowBuffer.close("invalidate");
     logger.logWarn(
         "Channel is invalidated, name={}, channel sequencer={}, row sequencer={}, message={}",
         getFullyQualifiedName(),
-        channelSequencer,
-        rowSequencer,
+        channelFlushContext.getChannelSequencer(),
+        channelState.getRowSequencer(),
         message);
   }
 
@@ -279,8 +226,8 @@ void markClosed() {
     logger.logInfo(
         "Channel is marked as closed, name={}, channel sequencer={}, row sequencer={}",
         getFullyQualifiedName(),
-        channelSequencer,
-        rowSequencer);
+        channelFlushContext.getChannelSequencer(),
+        channelState.getRowSequencer());
   }
 
   /**
@@ -340,15 +287,6 @@ public CompletableFuture<Void> close() {
             });
   }
 
-  /**
-   * Get the buffer allocator
-   *
-   * @return the buffer allocator
-   */
-  BufferAllocator getAllocator() {
-    return this.allocator;
-  }
-
   /**
    * Setup the column fields and vectors using the column metadata from the server
    *
@@ -500,12 +438,14 @@ private void checkValidation() {
     }
   }
 
-  OpenChannelRequest.OnErrorOption getOnErrorOption() {
-    return this.onErrorOption;
-  }
-
   /** Returns underlying channel's row buffer implementation. */
   RowBuffer<T> getRowBuffer() {
     return rowBuffer;
   }
+
+  /** Returns underlying channel's attributes. */
+  @VisibleForTesting
+  public ChannelFlushContext getChannelContext() {
+    return channelFlushContext;
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 8be3c4ed3..3e6ae6614 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -693,19 +693,20 @@ List<SnowflakeStreamingIngestChannelInternal<?>> verifyChannelsAreFullyCommitted
       for (int idx = 0; idx < channelsStatus.size(); idx++) {
         ChannelsStatusResponse.ChannelStatusResponseDTO channelStatus = channelsStatus.get(idx);
         SnowflakeStreamingIngestChannelInternal<?> channel = channels.get(idx);
+        long rowSequencer = channel.getChannelState().getRowSequencer();
         logger.logInfo(
             "Get channel status name={}, status={}, clientSequencer={}, rowSequencer={},"
                 + " offsetToken={}, persistedRowSequencer={}, persistedOffsetToken={}",
             channel.getName(),
             channelStatus.getStatusCode(),
             channel.getChannelSequencer(),
-            channel.getRowSequencer(),
-            channel.getOffsetToken(),
+            rowSequencer,
+            channel.getChannelState().getOffsetToken(),
             channelStatus.getPersistedRowSequencer(),
             channelStatus.getPersistedOffsetToken());
         if (channelStatus.getStatusCode() != RESPONSE_SUCCESS) {
           channelsWithError.add(channel);
-        } else if (!channelStatus.getPersistedRowSequencer().equals(channel.getRowSequencer())) {
+        } else if (!channelStatus.getPersistedRowSequencer().equals(rowSequencer)) {
           tempChannels.add(channel);
           tempChannelsStatus.add(channelStatus);
         }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
index 5785bd221..89d61dbe7 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
@@ -8,6 +8,7 @@
 import java.util.Map;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
+import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.pojo.ArrowType;
@@ -26,12 +27,9 @@ public void setupRowBuffer() {
   }
 
   ArrowRowBuffer createTestBuffer(OpenChannelRequest.OnErrorOption onErrorOption) {
-    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
-        new SnowflakeStreamingIngestClientInternal<>("client");
-    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel =
-        new SnowflakeStreamingIngestChannelInternal<>(
-            "channel", "db", "schema", "table", "0", 0L, 0L, client, "key", 1234L, onErrorOption);
-    return new ArrowRowBuffer(channel);
+    ChannelRuntimeState initialState = new ChannelRuntimeState("0", 0L, true);
+    return new ArrowRowBuffer(
+        onErrorOption, new RootAllocator(), "test.buffer", rs -> {}, initialState);
   }
 
   @Test
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 540cd5ea8..d80e21aab 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -100,7 +100,9 @@ private abstract static class TestContext<T> implements AutoCloseable {
     }
 
     ChannelData<T> flushChannel(String name) {
-      ChannelData<T> channelData = channels.get(name).getRowBuffer().flush();
+      SnowflakeStreamingIngestChannelInternal<T> channel = channels.get(name);
+      ChannelData<T> channelData = channel.getRowBuffer().flush();
+      channelData.setChannelContext(channel.getChannelContext());
       this.channelData.add(channelData);
       return channelData;
     }
@@ -324,15 +326,13 @@ TestContext<List<List<Object>>> create() {
   }
 
   TestContextFactory<?> testContextFactory;
-  TestContext<?> testContext;
 
   @Before
   public void setup() {
     java.security.Security.addProvider(new BouncyCastleProvider());
-    testContext = testContextFactory.create();
   }
 
-  private SnowflakeStreamingIngestChannelInternal<?> addChannel1() {
+  private SnowflakeStreamingIngestChannelInternal<?> addChannel1(TestContext<?> testContext) {
     return testContext
         .channelBuilder("channel1")
         .setDBName("db1")
@@ -344,7 +344,7 @@ private SnowflakeStreamingIngestChannelInternal<?> addChannel1() {
         .buildAndAdd();
   }
 
-  private SnowflakeStreamingIngestChannelInternal<?> addChannel2() {
+  private SnowflakeStreamingIngestChannelInternal<?> addChannel2(TestContext<?> testContext) {
     return testContext
         .channelBuilder("channel2")
         .setDBName("db1")
@@ -356,7 +356,7 @@ private SnowflakeStreamingIngestChannelInternal<?> addChannel2() {
         .buildAndAdd();
   }
 
-  private SnowflakeStreamingIngestChannelInternal<?> addChannel3() {
+  private SnowflakeStreamingIngestChannelInternal<?> addChannel3(TestContext<?> testContext) {
     return testContext
         .channelBuilder("channel3")
         .setDBName("db2")
@@ -394,6 +394,7 @@ private static ColumnMetadata createTestTextColumn() {
 
   @Test
   public void testGetFilePath() {
+    TestContext<?> testContext = testContextFactory.create();
     FlushService<?> flushService = testContext.flushService;
     Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
     String clientPrefix = "honk";
@@ -427,6 +428,7 @@ public void testGetFilePath() {
 
   @Test
   public void testFlush() throws Exception {
+    TestContext<?> testContext = testContextFactory.create();
     FlushService<?> flushService = testContext.flushService;
 
     // Nothing to flush
@@ -453,8 +455,9 @@ public void testFlush() throws Exception {
 
   @Test
   public void testBuildAndUpload() throws Exception {
-    SnowflakeStreamingIngestChannelInternal<?> channel1 = addChannel1();
-    SnowflakeStreamingIngestChannelInternal<?> channel2 = addChannel2();
+    TestContext<?> testContext = testContextFactory.create();
+    SnowflakeStreamingIngestChannelInternal<?> channel1 = addChannel1(testContext);
+    SnowflakeStreamingIngestChannelInternal<?> channel2 = addChannel2(testContext);
 
     List<ColumnMetadata> schema = Arrays.asList(createTestIntegerColumn(), createTestTextColumn());
     channel1.getRowBuffer().setupSchema(schema);
@@ -508,19 +511,19 @@ public void testBuildAndUpload() throws Exception {
 
     ChannelMetadata expectedChannel1Metadata =
         ChannelMetadata.builder()
-            .setOwningChannel(channel1)
+            .setOwningChannelFromContext(channel1.getChannelContext())
             .setRowSequencer(1L)
             .setOffsetToken("offset1")
             .build();
     ChannelMetadata expectedChannel2Metadata =
         ChannelMetadata.builder()
-            .setOwningChannel(channel2)
+            .setOwningChannelFromContext(channel2.getChannelContext())
             .setRowSequencer(10L)
             .setOffsetToken("offset2")
             .build();
     ChunkMetadata expectedChunkMetadata =
         ChunkMetadata.builder()
-            .setOwningTable(channel1)
+            .setOwningTableFromChannelContext(channel1.getChannelContext())
             .setChunkStartOffset(0L)
             .setChunkLength(248)
             .setChannelList(Arrays.asList(expectedChannel1Metadata, expectedChannel2Metadata))
@@ -580,8 +583,9 @@ public void testBuildAndUpload() throws Exception {
 
   @Test
   public void testBuildErrors() throws Exception {
-    SnowflakeStreamingIngestChannelInternal<?> channel1 = addChannel1();
-    SnowflakeStreamingIngestChannelInternal<?> channel3 = addChannel3();
+    TestContext<?> testContext = testContextFactory.create();
+    SnowflakeStreamingIngestChannelInternal<?> channel1 = addChannel1(testContext);
+    SnowflakeStreamingIngestChannelInternal<?> channel3 = addChannel3(testContext);
 
     List<ColumnMetadata> schema = Arrays.asList(createTestIntegerColumn(), createTestTextColumn());
     channel1.getRowBuffer().setupSchema(schema);
@@ -658,16 +662,18 @@ public void testInvalidateChannels() {
 
     ChannelData<StubChunkData> channel1Data = new ChannelData<>();
     ChannelData<StubChunkData> channel2Data = new ChannelData<>();
-    channel1Data.setChannel(channel1);
-    channel2Data.setChannel(channel1);
+    channel1Data.setChannelContext(channel1.getChannelContext());
+    channel2Data.setChannelContext(channel1.getChannelContext());
 
     channel1Data.setVectors(new StubChunkData());
     channel2Data.setVectors(new StubChunkData());
     innerData.add(channel1Data);
     innerData.add(channel2Data);
 
+    StreamingIngestStage stage = Mockito.mock(StreamingIngestStage.class);
+    Mockito.when(stage.getClientPrefix()).thenReturn("client_prefix");
     FlushService<StubChunkData> flushService =
-        new FlushService<>(client, channelCache, testContext.stage, false);
+        new FlushService<>(client, channelCache, stage, false);
     flushService.invalidateAllChannelsInBlob(blobData);
 
     Assert.assertFalse(channel1.isValid());
@@ -676,7 +682,8 @@ public void testInvalidateChannels() {
 
   @Test
   public void testBlobBuilder() throws Exception {
-    SnowflakeStreamingIngestChannelInternal<?> channel1 = addChannel1();
+    TestContext<?> testContext = testContextFactory.create();
+    SnowflakeStreamingIngestChannelInternal<?> channel1 = addChannel1(testContext);
 
     ObjectMapper mapper = new ObjectMapper();
     List<ChunkMetadata> chunksMetadataList = new ArrayList<>();
@@ -699,13 +706,13 @@ public void testBlobBuilder() throws Exception {
 
     ChannelMetadata channelMetadata =
         ChannelMetadata.builder()
-            .setOwningChannel(channel1)
+            .setOwningChannelFromContext(channel1.getChannelContext())
             .setRowSequencer(0L)
             .setOffsetToken("offset1")
             .build();
     ChunkMetadata chunkMetadata =
         ChunkMetadata.builder()
-            .setOwningTable(channel1)
+            .setOwningTableFromChannelContext(channel1.getChannelContext())
             .setChunkStartOffset(0L)
             .setChunkLength(dataSize)
             .setChannelList(Collections.singletonList(channelMetadata))
@@ -718,7 +725,7 @@ public void testBlobBuilder() throws Exception {
 
     final Constants.BdecVersion bdecVersion = ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT;
     byte[] blob =
-        BlobBuilder.build(chunksMetadataList, chunksDataList, checksum, dataSize, bdecVersion);
+        BlobBuilder.buildBlob(chunksMetadataList, chunksDataList, checksum, dataSize, bdecVersion);
 
     // Read the blob byte array back to valid the behavior
     InputStream input = new ByteArrayInputStream(blob);
@@ -770,6 +777,7 @@ public void testBlobBuilder() throws Exception {
 
   @Test
   public void testShutDown() throws Exception {
+    TestContext<?> testContext = testContextFactory.create();
     FlushService<?> flushService = testContext.flushService;
 
     Assert.assertFalse(flushService.buildUploadWorkers.isShutdown());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 8d28c0cb9..290c6416c 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -114,22 +114,9 @@ static List<ColumnMetadata> createSchema() {
   }
 
   private AbstractRowBuffer<?> createTestBuffer(OpenChannelRequest.OnErrorOption onErrorOption) {
-    return (AbstractRowBuffer<?>)
-        new SnowflakeStreamingIngestChannelInternal<>(
-                "channel",
-                "db",
-                "schema",
-                "table",
-                "0",
-                0L,
-                0L,
-                new SnowflakeStreamingIngestClientInternal<>("client"),
-                "key",
-                1234L,
-                onErrorOption,
-                bdecVersion,
-                new RootAllocator())
-            .getRowBuffer();
+    ChannelRuntimeState initialState = new ChannelRuntimeState("0", 0L, true);
+    return AbstractRowBuffer.createRowBuffer(
+        onErrorOption, new RootAllocator(), bdecVersion, "test.buffer", rs -> {}, initialState);
   }
 
   @Test
@@ -300,7 +287,7 @@ private void testStringLengthHelper(AbstractRowBuffer<?> rowBuffer) {
     row.put("colDecimal", 1.23);
     row.put("colChar", "1111111111111111111111"); // too big
 
-    if (rowBuffer.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.CONTINUE) {
+    if (rowBuffer.onErrorOption == OpenChannelRequest.OnErrorOption.CONTINUE) {
       response = rowBuffer.insertRows(Collections.singletonList(row), null);
       Assert.assertTrue(response.hasErrors());
       Assert.assertEquals(1, response.getErrorRowCount());
@@ -568,7 +555,7 @@ private void testE2ETimestampErrorsHelper(AbstractRowBuffer<?> innerBuffer) {
     row.put("COLTIMESTAMPLTZ_SB8", "1621899220");
     row.put("COLTIMESTAMPLTZ_SB16", "1621899220.1234567");
 
-    if (innerBuffer.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.CONTINUE) {
+    if (innerBuffer.onErrorOption == OpenChannelRequest.OnErrorOption.CONTINUE) {
       InsertValidationResponse response =
           innerBuffer.insertRows(Collections.singletonList(row), null);
       Assert.assertTrue(response.hasErrors());
@@ -882,7 +869,7 @@ private void testNullableCheckHelper(OpenChannelRequest.OnErrorOption onErrorOpt
     Assert.assertFalse(response.hasErrors());
 
     row.put("COLBOOLEAN", null);
-    if (innerBuffer.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.CONTINUE) {
+    if (innerBuffer.onErrorOption == OpenChannelRequest.OnErrorOption.CONTINUE) {
       response = innerBuffer.insertRows(Collections.singletonList(row), "1");
       Assert.assertTrue(response.hasErrors());
       Assert.assertEquals(
@@ -929,7 +916,7 @@ private void testMissingColumnCheckHelper(OpenChannelRequest.OnErrorOption onErr
 
     Map<String, Object> row2 = new HashMap<>();
     row2.put("COLBOOLEAN2", true);
-    if (innerBuffer.owningChannel.getOnErrorOption() == OpenChannelRequest.OnErrorOption.CONTINUE) {
+    if (innerBuffer.onErrorOption == OpenChannelRequest.OnErrorOption.CONTINUE) {
       response = innerBuffer.insertRows(Collections.singletonList(row2), "2");
       Assert.assertTrue(response.hasErrors());
       InsertValidationResponse.InsertError error = response.getInsertErrors().get(0);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 63e99cfff..847972376 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -111,9 +111,9 @@ public void testChannelFactorySuccess() {
     Assert.assertEquals(dbName, channel.getDBName());
     Assert.assertEquals(schemaName, channel.getSchemaName());
     Assert.assertEquals(tableName, channel.getTableName());
-    Assert.assertEquals(offsetToken, channel.getOffsetToken());
+    Assert.assertEquals(offsetToken, channel.getChannelState().getOffsetToken());
     Assert.assertEquals(channelSequencer, channel.getChannelSequencer());
-    Assert.assertEquals(rowSequencer + 1L, channel.incrementAndGetRowSequencer());
+    Assert.assertEquals(rowSequencer + 1L, channel.getChannelState().incrementAndGetRowSequencer());
     Assert.assertEquals(
         String.format("%s.%s.%s.%s", dbName, schemaName, tableName, name),
         channel.getFullyQualifiedName());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index ab48cc384..65d97d0b2 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -454,9 +454,9 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
 
     ChannelMetadata channelMetadata =
         ChannelMetadata.builder()
-            .setOwningChannel(channel)
-            .setRowSequencer(channel.incrementAndGetRowSequencer())
-            .setOffsetToken(channel.getOffsetToken())
+            .setOwningChannelFromContext(channel.getChannelContext())
+            .setRowSequencer(channel.getChannelState().incrementAndGetRowSequencer())
+            .setOffsetToken(channel.getChannelState().getOffsetToken())
             .build();
 
     Map<String, RowBufferStats> columnEps = new HashMap<>();
@@ -465,7 +465,7 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
 
     ChunkMetadata chunkMetadata =
         ChunkMetadata.builder()
-            .setOwningTable(channel)
+            .setOwningTableFromChannelContext(channel.getChannelContext())
             .setChunkStartOffset(0L)
             .setChunkLength(100)
             .setChannelList(Collections.singletonList(channelMetadata))
@@ -506,27 +506,27 @@ private Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> getRetryBlobMetadata(
 
     ChannelMetadata channelMetadata1 =
         ChannelMetadata.builder()
-            .setOwningChannel(channel1)
-            .setRowSequencer(channel1.incrementAndGetRowSequencer())
-            .setOffsetToken(channel1.getOffsetToken())
+            .setOwningChannelFromContext(channel1.getChannelContext())
+            .setRowSequencer(channel1.getChannelState().incrementAndGetRowSequencer())
+            .setOffsetToken(channel1.getChannelState().getOffsetToken())
             .build();
     ChannelMetadata channelMetadata2 =
         ChannelMetadata.builder()
-            .setOwningChannel(channel2)
-            .setRowSequencer(channel2.incrementAndGetRowSequencer())
-            .setOffsetToken(channel2.getOffsetToken())
+            .setOwningChannelFromContext(channel2.getChannelContext())
+            .setRowSequencer(channel2.getChannelState().incrementAndGetRowSequencer())
+            .setOffsetToken(channel2.getChannelState().getOffsetToken())
             .build();
     ChannelMetadata channelMetadata3 =
         ChannelMetadata.builder()
-            .setOwningChannel(channel3)
-            .setRowSequencer(channel3.incrementAndGetRowSequencer())
-            .setOffsetToken(channel3.getOffsetToken())
+            .setOwningChannelFromContext(channel3.getChannelContext())
+            .setRowSequencer(channel3.getChannelState().incrementAndGetRowSequencer())
+            .setOffsetToken(channel3.getChannelState().getOffsetToken())
             .build();
     ChannelMetadata channelMetadata4 =
         ChannelMetadata.builder()
-            .setOwningChannel(channel4)
-            .setRowSequencer(channel4.incrementAndGetRowSequencer())
-            .setOffsetToken(channel4.getOffsetToken())
+            .setOwningChannelFromContext(channel4.getChannelContext())
+            .setRowSequencer(channel4.getChannelState().incrementAndGetRowSequencer())
+            .setOffsetToken(channel4.getChannelState().getOffsetToken())
             .build();
 
     List<BlobMetadata> blobs = new ArrayList<>();
@@ -538,7 +538,7 @@ private Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> getRetryBlobMetadata(
     channels1.add(channelMetadata2);
     ChunkMetadata chunkMetadata1 =
         ChunkMetadata.builder()
-            .setOwningTable(channel1)
+            .setOwningTableFromChannelContext(channel1.getChannelContext())
             .setChunkStartOffset(0L)
             .setChunkLength(100)
             .setChannelList(channels1)
@@ -548,7 +548,7 @@ private Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> getRetryBlobMetadata(
             .build();
     ChunkMetadata chunkMetadata2 =
         ChunkMetadata.builder()
-            .setOwningTable(channel2)
+            .setOwningTableFromChannelContext(channel2.getChannelContext())
             .setChunkStartOffset(0L)
             .setChunkLength(100)
             .setChannelList(Collections.singletonList(channelMetadata3))
@@ -558,7 +558,7 @@ private Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> getRetryBlobMetadata(
             .build();
     ChunkMetadata chunkMetadata3 =
         ChunkMetadata.builder()
-            .setOwningTable(channel3)
+            .setOwningTableFromChannelContext(channel3.getChannelContext())
             .setChunkStartOffset(0L)
             .setChunkLength(100)
             .setChannelList(Collections.singletonList(channelMetadata4))

From 701824bf34a35a966a0c99a83d56f7fe09230e15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gloria=20Do=C3=A7i?= <gloria.doci@snowflake.com>
Date: Thu, 1 Dec 2022 12:51:48 +0100
Subject: [PATCH 096/356] SNOW-696155 fix binary values for Parquet (#292)

SNOW-696155 fix issue in Binary column value for Parquet
---
 .../streaming/internal/ParquetFlusher.java    |  6 ++-
 .../streaming/internal/ParquetRowBuffer.java  |  2 +-
 .../internal/ParquetValueParser.java          | 42 +++++++++++++------
 .../internal/ParquetValueParserTest.java      | 11 +++--
 4 files changed, 41 insertions(+), 20 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
index 3ea1ed44b..e5b496ff5 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
@@ -329,7 +329,11 @@ public void write(List<Object> values) {
               recordConsumer.addLong((long) val);
               break;
             case BINARY:
-              recordConsumer.addBinary(Binary.fromString((String) val));
+              Binary binVal =
+                  val instanceof String
+                      ? Binary.fromString((String) val)
+                      : Binary.fromConstantByteArray((byte[]) val);
+              recordConsumer.addBinary(binVal);
               break;
             case FIXED_LEN_BYTE_ARRAY:
               Binary binary = Binary.fromConstantByteArray((byte[]) val);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 1ef2e854b..009c2cb61 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -193,7 +193,7 @@ Object getVectorValueAt(String column, int index) {
       }
     }
     if (logicalType == ColumnLogicalType.BINARY && value != null) {
-      value = ((String) value).getBytes(StandardCharsets.UTF_8);
+      value = value instanceof String ? ((String) value).getBytes(StandardCharsets.UTF_8) : value;
     }
     return value;
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
index 671af1a5b..ebd958b75 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
@@ -95,9 +95,14 @@ static ParquetBufferValue parseColumnValueToParquet(
           size = 8;
           break;
         case BINARY:
-          String str = getBinaryValue(value, stats, columnMetadata);
-          value = str;
-          size = str.getBytes().length;
+          if (logicalType == AbstractRowBuffer.ColumnLogicalType.BINARY) {
+            value = getBinaryValueForLogicalBinary(value, stats, columnMetadata);
+            size = ((byte[]) value).length;
+          } else {
+            String str = getBinaryValue(value, stats, columnMetadata);
+            value = str;
+            size = str.getBytes().length;
+          }
           break;
         case FIXED_LEN_BYTE_ARRAY:
           BigInteger intRep =
@@ -277,12 +282,12 @@ static byte[] getSb16Bytes(BigInteger intRep) {
   }
 
   /**
-   * Converts an object, string or binary value to its byte array representation.
+   * Converts an object or string to its byte array representation.
    *
    * @param value value to parse
    * @param stats column stats to update
    * @param columnMetadata column metadata
-   * @return string (byte array) representation
+   * @return string representation
    */
   private static String getBinaryValue(
       Object value, RowBufferStats stats, ColumnMetadata columnMetadata) {
@@ -304,13 +309,6 @@ private static String getBinaryValue(
           throw new SFException(
               ErrorCode.UNKNOWN_DATA_TYPE, logicalType, columnMetadata.getPhysicalType());
       }
-    } else if (logicalType == AbstractRowBuffer.ColumnLogicalType.BINARY) {
-      String maxLengthString = columnMetadata.getLength().toString();
-      byte[] bytes =
-          DataValidationUtil.validateAndParseBinary(
-              value, Optional.of(maxLengthString).map(Integer::parseInt));
-      str = new String(bytes, StandardCharsets.UTF_8);
-      stats.addStrValue(str);
     } else {
       String maxLengthString = columnMetadata.getLength().toString();
       str =
@@ -320,4 +318,24 @@ private static String getBinaryValue(
     }
     return str;
   }
+  /**
+   * Converts a binary value to its byte array representation.
+   *
+   * @param value value to parse
+   * @param stats column stats to update
+   * @param columnMetadata column metadata
+   * @return byte array representation
+   */
+  private static byte[] getBinaryValueForLogicalBinary(
+      Object value, RowBufferStats stats, ColumnMetadata columnMetadata) {
+    String maxLengthString = columnMetadata.getLength().toString();
+    byte[] bytes =
+        DataValidationUtil.validateAndParseBinary(
+            value, Optional.of(maxLengthString).map(Integer::parseInt));
+
+    String str = new String(bytes, StandardCharsets.UTF_8);
+    stats.addStrValue(str);
+
+    return bytes;
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
index 3dc313dc6..199ca5124 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
@@ -235,15 +235,15 @@ public void parseValueBinary() {
     RowBufferStats rowBufferStats = new RowBufferStats();
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            "Length7".getBytes(), testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
+            "1234abcd".getBytes(), testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
         .rowBufferStats(rowBufferStats)
-        .expectedValueClass(String.class)
-        .expectedParsedValue("Length7")
-        .expectedSize(7.0f)
-        .expectedMinMax("Length7")
+        .expectedValueClass(byte[].class)
+        .expectedParsedValue("1234abcd".getBytes())
+        .expectedSize(8.0f)
+        .expectedMinMax("1234abcd")
         .assertMatches();
   }
 
@@ -543,7 +543,6 @@ public ParquetValueParserAssertionBuilder expectedMinMax(Object minMaxStat) {
 
     void assertMatches() {
       Assert.assertEquals(valueClass, parquetBufferValue.getValue().getClass());
-      System.out.println("parquetBufferValue = " + parquetBufferValue.getValue().toString());
       if (valueClass.equals(byte[].class)) {
         Assert.assertArrayEquals((byte[]) value, (byte[]) parquetBufferValue.getValue());
       } else {

From b1c378836431d205b953a03d0bf25b35a08325bd Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Fri, 2 Dec 2022 18:04:47 +0100
Subject: [PATCH 097/356] SNOW-647377 Support quoted column names (#293)

This PR implements initial support for quoted column names.
Internal column name is obtained from display name by calling
unquote. This will be changed later to receive it from the
server side. User input is matched with column name by calling
unquote on both and comparing.

The following names are used in these situation:

* Internal data structures in the SDK => Internal name
* Arrow buffer, Parquet buffer => Internal name
* BufferStats, EpInfo => Display name
* Extra columns in user input => Original (unquoted) user input
* Missing columns in user input => Display name

Testing
Integration tests have been added.
---
 .../streaming/internal/AbstractRowBuffer.java |  19 +-
 .../streaming/internal/ArrowRowBuffer.java    |  12 +-
 .../streaming/internal/ColumnMetadata.java    |   6 +
 .../streaming/internal/LiteralQuoteUtils.java |  65 +++++
 .../streaming/internal/ParquetRowBuffer.java  |  12 +-
 .../internal/ParquetTypeGenerator.java        |   4 +-
 .../streaming/internal/RowBufferStats.java    |  16 +-
 .../streaming/internal/ArrowBufferTest.java   |  30 +--
 .../streaming/internal/ChannelDataTest.java   |  14 +-
 .../streaming/internal/FlushServiceTest.java  |   6 +-
 .../internal/ParquetTypeGeneratorTest.java    |  30 +--
 .../internal/ParquetValueParserTest.java      |  36 +--
 .../internal/RowBufferStatsTest.java          |  66 ++---
 .../streaming/internal/RowBufferTest.java     |  28 ++-
 .../SnowflakeStreamingIngestClientTest.java   |   4 +-
 .../streaming/internal/StreamingIngestIT.java |  82 +++++++
 .../datatypes/AbstractDataTypeTest.java       |   7 +-
 .../streaming/internal/it/ColumnNamesIT.java  | 231 ++++++++++++++++++
 18 files changed, 533 insertions(+), 135 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/LiteralQuoteUtils.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/it/ColumnNamesIT.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 4a49ab114..0a61aa058 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -21,7 +21,6 @@
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.Utils;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.util.VisibleForTesting;
 
@@ -222,7 +221,7 @@ Set<String> verifyInputColumns(
       Map<String, Object> row, InsertValidationResponse.InsertError error) {
     Map<String, String> inputColNamesMap =
         row.keySet().stream()
-            .collect(Collectors.toMap(AbstractRowBuffer::formatColumnName, value -> value));
+            .collect(Collectors.toMap(LiteralQuoteUtils::unquoteColumnName, value -> value));
 
     // Check for extra columns in the row
     List<String> extraCols = new ArrayList<>();
@@ -246,7 +245,7 @@ Set<String> verifyInputColumns(
     List<String> missingCols = new ArrayList<>();
     for (String columnName : this.nonNullableFieldNames) {
       if (!inputColNamesMap.containsKey(columnName)) {
-        missingCols.add(columnName);
+        missingCols.add(statsMap.get(columnName).getColumnDisplayName());
       }
     }
 
@@ -448,7 +447,8 @@ void reset() {
     this.rowCount = 0;
     this.bufferSize = 0F;
     this.statsMap.replaceAll(
-        (key, value) -> new RowBufferStats(value.getCollationDefinitionString()));
+        (key, value) ->
+            new RowBufferStats(value.getColumnDisplayName(), value.getCollationDefinitionString()));
   }
 
   /** Get buffered data snapshot for later flushing. */
@@ -460,14 +460,6 @@ void reset() {
   @VisibleForTesting
   abstract int getTempRowCount();
 
-  /** Normalize the column name, given with the inserted row, to send to server side. */
-  static String formatColumnName(String columnName) {
-    Utils.assertStringNotNullOrEmpty("invalid column name", columnName);
-    return (columnName.charAt(0) == '"' && columnName.charAt(columnName.length() - 1) == '"')
-        ? columnName
-        : columnName.toUpperCase();
-  }
-
   /**
    * Given a set of col names to stats, build the right ep info
    *
@@ -480,8 +472,7 @@ static EpInfo buildEpInfoFromStats(long rowCount, Map<String, RowBufferStats> co
     for (Map.Entry<String, RowBufferStats> colStat : colStats.entrySet()) {
       RowBufferStats stat = colStat.getValue();
       FileColumnProperties dto = new FileColumnProperties(stat);
-
-      String colName = colStat.getKey();
+      String colName = colStat.getValue().getColumnDisplayName();
       epInfo.getColumnEps().put(colName, dto);
     }
     return epInfo;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 9b6001d43..6776b310c 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -105,14 +105,16 @@ public void setupSchema(List<ColumnMetadata> columns) {
       if (!field.isNullable()) {
         addNonNullableFieldName(field.getName());
       }
-      this.fields.put(column.getName(), field);
+      this.fields.put(column.getInternalName(), field);
       vectors.add(vector);
-      this.statsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
+      this.statsMap.put(
+          column.getInternalName(), new RowBufferStats(column.getName(), column.getCollation()));
 
       if (onErrorOption == OpenChannelRequest.OnErrorOption.ABORT) {
         FieldVector tempVector = field.createVector(this.allocator);
         tempVectors.add(tempVector);
-        this.tempStatsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
+        this.tempStatsMap.put(
+            column.getInternalName(), new RowBufferStats(column.getName(), column.getCollation()));
       }
     }
 
@@ -336,7 +338,7 @@ Field buildField(ColumnMetadata column) {
 
     // Create the corresponding column field base on the column data type
     fieldType = new FieldType(column.getNullable(), arrowType, null, metadata);
-    return new Field(column.getName(), fieldType, children);
+    return new Field(column.getInternalName(), fieldType, children);
   }
 
   @Override
@@ -440,7 +442,7 @@ private float convertRowToArrow(
     float rowBufferSize = 0F;
     for (Map.Entry<String, Object> entry : row.entrySet()) {
       rowBufferSize += 0.125; // 1/8 for null value bitmap
-      String columnName = formatColumnName(entry.getKey());
+      String columnName = LiteralQuoteUtils.unquoteColumnName(entry.getKey());
       Object value = entry.getValue();
       Field field = this.fields.get(columnName);
       Utils.assertNotNull("Arrow column field", field);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ColumnMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/ColumnMetadata.java
index 108035a02..b9779b41f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ColumnMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ColumnMetadata.java
@@ -11,6 +11,7 @@
 /** Column metadata for each column in a Snowflake table */
 class ColumnMetadata {
   private String name;
+  private String internalName;
   private String type;
   private String logicalType;
   private String physicalType;
@@ -24,6 +25,7 @@ class ColumnMetadata {
   @JsonProperty("name")
   void setName(String name) {
     this.name = name;
+    this.internalName = LiteralQuoteUtils.unquoteColumnName(name);
   }
 
   String getName() {
@@ -111,6 +113,10 @@ boolean getNullable() {
     return this.nullable;
   }
 
+  String getInternalName() {
+    return internalName;
+  }
+
   @Override
   public String toString() {
     Map<String, Object> map = new HashMap<>();
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/LiteralQuoteUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/LiteralQuoteUtils.java
new file mode 100644
index 000000000..285a62315
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/LiteralQuoteUtils.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+package net.snowflake.ingest.streaming.internal;
+
+/**
+ * Util class to normalise literals to match server side metadata.
+ *
+ * <p>Note: The methods in this class have to be kept in sync with the respective methods on server
+ * side.
+ */
+class LiteralQuoteUtils {
+
+  /**
+   * Unquote SQL literal.
+   *
+   * <p>Normalises the column name to how it is stored internally. This function needs to keep in
+   * sync with server side normalisation. The reason, we do it here, is to store the normalised
+   * column name in BDEC file metadata.
+   *
+   * @param columnName column name literal to unquote
+   * @return unquoted literal
+   */
+  static String unquoteColumnName(String columnName) {
+    int length = columnName.length();
+
+    if (length == 0) {
+      return columnName;
+    }
+
+    // If this is an identifier that starts and ends with double quotes,
+    // remove them - accounting for escaped double quotes.
+    // Differs from the second condition in that this one allows repeated
+    // double quotes
+    if (columnName.charAt(0) == '"'
+        && (length >= 2
+            && columnName.charAt(length - 1) == '"'
+            &&
+            // Condition that the string contains no single double-quotes
+            // but allows repeated double-quotes
+            !columnName.substring(1, length - 1).replace("\"\"", "").contains("\""))) {
+      // Remove quotes and turn escaped double-quotes to single double-quotes
+      return columnName.substring(1, length - 1).replace("\"\"", "\"");
+    }
+
+    // If this is an identifier that starts and ends with double quotes,
+    // remove them. Internal single double-quotes are not allowed.
+    else if (columnName.charAt(0) == '"'
+        && (length >= 2
+            && columnName.charAt(length - 1) == '"'
+            && !columnName.substring(1, length - 1).contains("\""))) {
+      // Remove the quotes
+      return columnName.substring(1, length - 1);
+    }
+
+    // unquoted string that can have escaped spaces
+    else {
+      // replace escaped spaces in unquoted name
+      if (columnName.contains("\\ ")) {
+        columnName = columnName.replace("\\ ", " ");
+      }
+      return columnName.toUpperCase();
+    }
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 009c2cb61..1e918e061 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -70,14 +70,16 @@ public void setupSchema(List<ColumnMetadata> columns) {
           ParquetTypeGenerator.generateColumnParquetTypeInfo(column, id);
       parquetTypes.add(typeInfo.getParquetType());
       this.metadata.putAll(typeInfo.getMetadata());
-      fieldIndex.put(column.getName(), new Pair<>(column, parquetTypes.size() - 1));
+      fieldIndex.put(column.getInternalName(), new Pair<>(column, parquetTypes.size() - 1));
       if (!column.getNullable()) {
-        addNonNullableFieldName(column.getName());
+        addNonNullableFieldName(column.getInternalName());
       }
-      this.statsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
+      this.statsMap.put(
+          column.getInternalName(), new RowBufferStats(column.getName(), column.getCollation()));
 
       if (onErrorOption == OpenChannelRequest.OnErrorOption.ABORT) {
-        this.tempStatsMap.put(column.getName(), new RowBufferStats(column.getCollation()));
+        this.tempStatsMap.put(
+            column.getInternalName(), new RowBufferStats(column.getName(), column.getCollation()));
       }
 
       id++;
@@ -127,7 +129,7 @@ private float addRow(
     for (Map.Entry<String, Object> entry : row.entrySet()) {
       String key = entry.getKey();
       Object value = entry.getValue();
-      String columnName = formatColumnName(key);
+      String columnName = LiteralQuoteUtils.unquoteColumnName(key);
       int colIndex = fieldIndex.get(columnName).getSecond();
       RowBufferStats stats = statsMap.get(columnName);
       ColumnMetadata column = fieldIndex.get(columnName).getFirst();
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
index 149ce766c..933909c2b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
@@ -67,7 +67,7 @@ static ParquetTypeInfo generateColumnParquetTypeInfo(ColumnMetadata column, int
     ParquetTypeInfo res = new ParquetTypeInfo();
     Type parquetType;
     Map<String, String> metadata = new HashMap<>();
-    String name = column.getName();
+    String name = column.getInternalName();
 
     AbstractRowBuffer.ColumnPhysicalType physicalType;
     AbstractRowBuffer.ColumnLogicalType logicalType;
@@ -172,7 +172,7 @@ private static Type getFixedColumnParquetType(
       int id,
       AbstractRowBuffer.ColumnPhysicalType physicalType,
       Type.Repetition repetition) {
-    String name = column.getName();
+    String name = column.getInternalName();
     // the LogicalTypeAnnotation.DecimalLogicalTypeAnnotation is used by server side scanner
     // to discover data type scale and precision
     LogicalTypeAnnotation.DecimalLogicalTypeAnnotation decimalLogicalTypeAnnotation =
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
index 54395b35c..2a6503084 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
@@ -306,11 +306,14 @@ byte[] performConversion(String input) {
   private long currentMaxLength;
   private CollationDefinition collationDefinition;
   private final String collationDefinitionString;
+  /** Display name is required for the registration endpoint */
+  private final String columnDisplayName;
 
   private static final int MAX_LOB_LEN = 32;
 
   /** Creates empty stats */
-  RowBufferStats(String collationDefinitionString) {
+  RowBufferStats(String columnDisplayName, String collationDefinitionString) {
+    this.columnDisplayName = columnDisplayName;
     this.collationDefinitionString = collationDefinitionString;
     if (collationDefinitionString != null) {
       this.collationDefinition = new CollationDefinition(collationDefinitionString);
@@ -318,8 +321,8 @@ byte[] performConversion(String input) {
     reset();
   }
 
-  RowBufferStats() {
-    this(null);
+  RowBufferStats(String columnDisplayName) {
+    this(columnDisplayName, null);
   }
 
   void reset() {
@@ -352,7 +355,8 @@ static RowBufferStats getCombinedStats(RowBufferStats left, RowBufferStats right
               "left=%s, right=%s",
               left.getCollationDefinitionString(), right.getCollationDefinitionString()));
     }
-    RowBufferStats combined = new RowBufferStats(left.getCollationDefinitionString());
+    RowBufferStats combined =
+        new RowBufferStats(left.columnDisplayName, left.getCollationDefinitionString());
 
     if (left.currentMinIntValue != null) {
       combined.addIntValue(left.currentMinIntValue);
@@ -524,6 +528,10 @@ String getCollationDefinitionString() {
     return collationDefinitionString;
   }
 
+  String getColumnDisplayName() {
+    return columnDisplayName;
+  }
+
   /**
    * Compares two byte arrays lexicographically. If the two arrays share a common prefix then the
    * lexicographic comparison is the result of comparing two elements, as if by Byte.compare(byte,
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
index 89d61dbe7..167e0d1b1 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
@@ -64,7 +64,7 @@ public void buildFieldFixedSB1() {
 
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
-    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals("TESTCOL", result.getName());
     Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.TINYINT.getType());
     Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB1");
     Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
@@ -85,7 +85,7 @@ public void buildFieldFixedSB2() {
 
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
-    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals("TESTCOL", result.getName());
     Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.SMALLINT.getType());
     Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB2");
     Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
@@ -106,7 +106,7 @@ public void buildFieldFixedSB4() {
 
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
-    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals("TESTCOL", result.getName());
     Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.INT.getType());
     Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB4");
     Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
@@ -126,7 +126,7 @@ public void buildFieldFixedSB8() {
 
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
-    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals("TESTCOL", result.getName());
     Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.BIGINT.getType());
     Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
     Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
@@ -149,7 +149,7 @@ public void buildFieldFixedSB16() {
     ArrowType expectedType =
         new ArrowType.Decimal(testCol.getPrecision(), testCol.getScale(), DECIMAL_BIT_WIDTH);
 
-    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals("TESTCOL", result.getName());
     Assert.assertEquals(result.getFieldType().getType(), expectedType);
     Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB16");
     Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
@@ -169,7 +169,7 @@ public void buildFieldLobVariant() {
 
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
-    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals("TESTCOL", result.getName());
     Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.VARCHAR.getType());
     Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "LOB");
     Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
@@ -189,7 +189,7 @@ public void buildFieldTimestampNtzSB8() {
 
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
-    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals("TESTCOL", result.getName());
     Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.BIGINT.getType());
     Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
     Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
@@ -209,7 +209,7 @@ public void buildFieldTimestampNtzSB16() {
 
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
-    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals("TESTCOL", result.getName());
     Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.STRUCT.getType());
     Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB16");
     Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
@@ -233,7 +233,7 @@ public void buildFieldTimestampTzSB8() {
 
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
-    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals("TESTCOL", result.getName());
     Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.STRUCT.getType());
     Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
     Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
@@ -257,7 +257,7 @@ public void buildFieldTimestampTzSB16() {
 
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
-    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals("TESTCOL", result.getName());
     Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.STRUCT.getType());
     Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB16");
     Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
@@ -283,7 +283,7 @@ public void buildFieldTimestampDate() {
 
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
-    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals("TESTCOL", result.getName());
     Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.DATEDAY.getType());
     Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
     Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
@@ -303,7 +303,7 @@ public void buildFieldTimeSB4() {
 
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
-    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals("TESTCOL", result.getName());
     Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.INT.getType());
     Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB4");
     Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
@@ -323,7 +323,7 @@ public void buildFieldTimeSB8() {
 
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
-    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals("TESTCOL", result.getName());
     Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.BIGINT.getType());
     Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
     Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
@@ -343,7 +343,7 @@ public void buildFieldBoolean() {
 
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
-    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals("TESTCOL", result.getName());
     Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.BIT.getType());
     Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "BINARY");
     Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
@@ -363,7 +363,7 @@ public void buildFieldRealSB16() {
 
     Field result = this.rowBufferOnErrorContinue.buildField(testCol);
 
-    Assert.assertEquals("testCol", result.getName());
+    Assert.assertEquals("TESTCOL", result.getName());
     Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.FLOAT8.getType());
     Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB16");
     Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelDataTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelDataTest.java
index cc072e070..f429c0d84 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelDataTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelDataTest.java
@@ -13,7 +13,7 @@ public class ChannelDataTest {
   @Test
   public void testGetCombinedColumnStatsMapNulls() {
     Map<String, RowBufferStats> left = new HashMap<>();
-    RowBufferStats leftStats1 = new RowBufferStats();
+    RowBufferStats leftStats1 = new RowBufferStats("COL1");
     left.put("one", leftStats1);
     leftStats1.addIntValue(new BigInteger("10"));
 
@@ -42,12 +42,12 @@ public void testGetCombinedColumnStatsMapNulls() {
   @Test
   public void testGetCombinedColumnStatsMapMissingColumn() {
     Map<String, RowBufferStats> left = new HashMap<>();
-    RowBufferStats leftStats1 = new RowBufferStats();
+    RowBufferStats leftStats1 = new RowBufferStats("COL1");
     left.put("one", leftStats1);
     leftStats1.addIntValue(new BigInteger("10"));
 
     Map<String, RowBufferStats> right = new HashMap<>();
-    RowBufferStats rightStats1 = new RowBufferStats();
+    RowBufferStats rightStats1 = new RowBufferStats("COL1");
     right.put("foo", rightStats1);
     rightStats1.addIntValue(new BigInteger("11"));
 
@@ -77,10 +77,10 @@ public void testGetCombinedColumnStatsMap() {
     Map<String, RowBufferStats> left = new HashMap<>();
     Map<String, RowBufferStats> right = new HashMap<>();
 
-    RowBufferStats leftStats1 = new RowBufferStats();
-    RowBufferStats rightStats1 = new RowBufferStats();
-    RowBufferStats leftStats2 = new RowBufferStats();
-    RowBufferStats rightStats2 = new RowBufferStats();
+    RowBufferStats leftStats1 = new RowBufferStats("COL1");
+    RowBufferStats rightStats1 = new RowBufferStats("COL1");
+    RowBufferStats leftStats2 = new RowBufferStats("COL1");
+    RowBufferStats rightStats2 = new RowBufferStats("COL1");
 
     left.put("one", leftStats1);
     left.put("two", leftStats2);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index d80e21aab..ce9a44d4d 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -483,8 +483,8 @@ public void testBuildAndUpload() throws Exception {
     Map<String, RowBufferStats> eps1 = new HashMap<>();
     Map<String, RowBufferStats> eps2 = new HashMap<>();
 
-    RowBufferStats stats1 = new RowBufferStats();
-    RowBufferStats stats2 = new RowBufferStats();
+    RowBufferStats stats1 = new RowBufferStats("COL1");
+    RowBufferStats stats2 = new RowBufferStats("COL1");
 
     eps1.put("one", stats1);
     eps2.put("one", stats2);
@@ -696,7 +696,7 @@ public void testBlobBuilder() throws Exception {
 
     Map<String, RowBufferStats> eps1 = new HashMap<>();
 
-    RowBufferStats stats1 = new RowBufferStats();
+    RowBufferStats stats1 = new RowBufferStats("COL1");
 
     eps1.put("one", stats1);
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
index a477ac977..0f6bf60c7 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
@@ -22,7 +22,7 @@ public void buildFieldFixedSB1() {
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
     ParquetTypeInfoAssertionBuilder.newBuilder()
         .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
+        .expectedFieldName("TESTCOL")
         .expectedFieldId(0)
         .expectedTypeLength(4)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
@@ -49,7 +49,7 @@ public void buildFieldFixedSB2() {
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
     ParquetTypeInfoAssertionBuilder.newBuilder()
         .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
+        .expectedFieldName("TESTCOL")
         .expectedFieldId(0)
         .expectedTypeLength(4)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
@@ -76,7 +76,7 @@ public void buildFieldFixedSB4() {
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
     ParquetTypeInfoAssertionBuilder.newBuilder()
         .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
+        .expectedFieldName("TESTCOL")
         .expectedFieldId(0)
         .expectedTypeLength(4)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
@@ -103,7 +103,7 @@ public void buildFieldFixedSB8() {
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
     ParquetTypeInfoAssertionBuilder.newBuilder()
         .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
+        .expectedFieldName("TESTCOL")
         .expectedFieldId(0)
         .expectedTypeLength(8)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
@@ -130,7 +130,7 @@ public void buildFieldFixedSB16() {
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
     ParquetTypeInfoAssertionBuilder.newBuilder()
         .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
+        .expectedFieldName("TESTCOL")
         .expectedFieldId(0)
         .expectedTypeLength(16)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
@@ -157,7 +157,7 @@ public void buildFieldLobVariant() {
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
     ParquetTypeInfoAssertionBuilder.newBuilder()
         .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
+        .expectedFieldName("TESTCOL")
         .expectedFieldId(0)
         .expectedTypeLength(0)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.BINARY)
@@ -185,7 +185,7 @@ public void buildFieldTimestampNtzSB8() {
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
     ParquetTypeInfoAssertionBuilder.newBuilder()
         .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
+        .expectedFieldName("TESTCOL")
         .expectedFieldId(0)
         .expectedTypeLength(8)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
@@ -211,7 +211,7 @@ public void buildFieldTimestampNtzSB16() {
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
     ParquetTypeInfoAssertionBuilder.newBuilder()
         .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
+        .expectedFieldName("TESTCOL")
         .expectedFieldId(0)
         .expectedTypeLength(16)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
@@ -237,7 +237,7 @@ public void buildFieldTimestampTzSB8() {
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
     ParquetTypeInfoAssertionBuilder.newBuilder()
         .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
+        .expectedFieldName("TESTCOL")
         .expectedFieldId(0)
         .expectedTypeLength(8)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
@@ -263,7 +263,7 @@ public void buildFieldTimestampTzSB16() {
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
     ParquetTypeInfoAssertionBuilder.newBuilder()
         .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
+        .expectedFieldName("TESTCOL")
         .expectedFieldId(0)
         .expectedTypeLength(16)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
@@ -289,7 +289,7 @@ public void buildFieldDate() {
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
     ParquetTypeInfoAssertionBuilder.newBuilder()
         .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
+        .expectedFieldName("TESTCOL")
         .expectedFieldId(0)
         .expectedTypeLength(0)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
@@ -315,7 +315,7 @@ public void buildFieldTimeSB4() {
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
     ParquetTypeInfoAssertionBuilder.newBuilder()
         .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
+        .expectedFieldName("TESTCOL")
         .expectedFieldId(0)
         .expectedTypeLength(4)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
@@ -341,7 +341,7 @@ public void buildFieldTimeSB8() {
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
     ParquetTypeInfoAssertionBuilder.newBuilder()
         .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
+        .expectedFieldName("TESTCOL")
         .expectedFieldId(0)
         .expectedTypeLength(8)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
@@ -367,7 +367,7 @@ public void buildFieldBoolean() {
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
     ParquetTypeInfoAssertionBuilder.newBuilder()
         .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
+        .expectedFieldName("TESTCOL")
         .expectedFieldId(0)
         .expectedTypeLength(0)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.BOOLEAN)
@@ -393,7 +393,7 @@ public void buildFieldRealSB16() {
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
     ParquetTypeInfoAssertionBuilder.newBuilder()
         .typeInfo(typeInfo)
-        .expectedFieldName("testCol")
+        .expectedFieldName("TESTCOL")
         .expectedFieldId(0)
         .expectedTypeLength(0)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.DOUBLE)
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
index 199ca5124..e580cc9b6 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
@@ -22,7 +22,7 @@ public void parseValueFixedSB1ToInt32() {
             .nullable(true)
             .build();
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
             12, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
@@ -48,7 +48,7 @@ public void parseValueFixedSB2ToInt32() {
             .nullable(true)
             .build();
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
             1234, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
@@ -74,7 +74,7 @@ public void parseValueFixedSB4ToInt32() {
             .nullable(true)
             .build();
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
             123456789, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
@@ -100,7 +100,7 @@ public void parseValueFixedSB8ToInt64() {
             .nullable(true)
             .build();
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
             123456789987654321L, testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats);
@@ -126,7 +126,7 @@ public void parseValueFixedSB16ToByteArray() {
             .nullable(true)
             .build();
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
             new BigDecimal("91234567899876543219876543211234567891"),
@@ -157,7 +157,7 @@ public void parseValueFixedDecimalToInt32() {
             .nullable(true)
             .build();
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
             new BigDecimal("12345.54321"),
@@ -184,7 +184,7 @@ public void parseValueDouble() {
             .nullable(true)
             .build();
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
             12345.54321d, testCol, PrimitiveType.PrimitiveTypeName.DOUBLE, rowBufferStats);
@@ -208,7 +208,7 @@ public void parseValueBoolean() {
             .nullable(true)
             .build();
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
             true, testCol, PrimitiveType.PrimitiveTypeName.BOOLEAN, rowBufferStats);
@@ -232,7 +232,7 @@ public void parseValueBinary() {
             .nullable(true)
             .build();
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
             "1234abcd".getBytes(), testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
@@ -268,7 +268,7 @@ private void testJsonWithLogicalType(String logicalType) {
     String var =
         "{\"key1\":-879869596,\"key2\":\"value2\",\"key3\":null,"
             + "\"key4\":{\"key41\":0.032437,\"key42\":\"value42\",\"key43\":null}}";
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
             var, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
@@ -297,7 +297,7 @@ public void parseValueArrayToBinary() {
     input.put("b", "2");
     input.put("c", "3");
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
             input, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
@@ -325,7 +325,7 @@ public void parseValueTimestampNtzSB4Error() {
             .nullable(true)
             .build();
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     SFException exception =
         Assert.assertThrows(
             SFException.class,
@@ -350,7 +350,7 @@ public void parseValueTimestampNtzSB8ToINT64() {
             .nullable(true)
             .build();
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
             "2013-04-28 20:57:01.000",
@@ -378,7 +378,7 @@ public void parseValueTimestampNtzSB16ToByteArray() {
             .scale(9) // nanos
             .build();
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
             "2022-09-18 22:05:07.123456789",
@@ -407,7 +407,7 @@ public void parseValueDateToInt32() {
             .nullable(true)
             .build();
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
             "2021-01-01", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
@@ -432,7 +432,7 @@ public void parseValueTimeSB4ToInt32() {
             .nullable(true)
             .build();
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
             "01:00:00", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
@@ -457,7 +457,7 @@ public void parseValueTimeSB8ToInt64() {
             .nullable(true)
             .build();
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
             "01:00:00.123", testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats);
@@ -482,7 +482,7 @@ public void parseValueTimeSB16Error() {
             .nullable(true)
             .build();
 
-    RowBufferStats rowBufferStats = new RowBufferStats();
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     SFException exception =
         Assert.assertThrows(
             SFException.class,
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferStatsTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferStatsTest.java
index fe4027fc0..fa3a38c07 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferStatsTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferStatsTest.java
@@ -8,17 +8,17 @@ public class RowBufferStatsTest {
 
   @Test
   public void testCollationStates() throws Exception {
-    RowBufferStats ai = new RowBufferStats("en-ai");
-    RowBufferStats as = new RowBufferStats("en-as");
-    RowBufferStats pi = new RowBufferStats("en-pi");
-    RowBufferStats ps = new RowBufferStats("en-ps");
-    RowBufferStats fu = new RowBufferStats("en-fu");
-    RowBufferStats fl = new RowBufferStats("en-fl");
-    RowBufferStats lower = new RowBufferStats("lower");
-    RowBufferStats upper = new RowBufferStats("upper");
-    RowBufferStats ltrim = new RowBufferStats("ltrim");
-    RowBufferStats rtrim = new RowBufferStats("rtrim");
-    RowBufferStats trim = new RowBufferStats("trim");
+    RowBufferStats ai = new RowBufferStats("COL1", "en-ai");
+    RowBufferStats as = new RowBufferStats("COL1", "en-as");
+    RowBufferStats pi = new RowBufferStats("COL1", "en-pi");
+    RowBufferStats ps = new RowBufferStats("COL1", "en-ps");
+    RowBufferStats fu = new RowBufferStats("COL1", "en-fu");
+    RowBufferStats fl = new RowBufferStats("COL1", "en-fl");
+    RowBufferStats lower = new RowBufferStats("COL1", "lower");
+    RowBufferStats upper = new RowBufferStats("COL1", "upper");
+    RowBufferStats ltrim = new RowBufferStats("COL1", "ltrim");
+    RowBufferStats rtrim = new RowBufferStats("COL1", "rtrim");
+    RowBufferStats trim = new RowBufferStats("COL1", "trim");
 
     // Accents
     ai.addStrValue("a");
@@ -69,7 +69,7 @@ public void testCollationStates() throws Exception {
 
   @Test
   public void testEmptyState() throws Exception {
-    RowBufferStats stats = new RowBufferStats();
+    RowBufferStats stats = new RowBufferStats("COL1");
 
     Assert.assertNull(stats.getCollationDefinitionString());
     Assert.assertNull(stats.getCurrentMinRealValue());
@@ -85,7 +85,7 @@ public void testEmptyState() throws Exception {
 
   @Test
   public void testMinMaxStrNonCol() throws Exception {
-    RowBufferStats stats = new RowBufferStats();
+    RowBufferStats stats = new RowBufferStats("COL1");
 
     stats.addStrValue("bob");
     Assert.assertEquals("bob", stats.getCurrentMinColStrValue());
@@ -112,7 +112,7 @@ public void testMinMaxStrNonCol() throws Exception {
 
   @Test
   public void testStrTruncation() throws Exception {
-    RowBufferStats stats = new RowBufferStats();
+    RowBufferStats stats = new RowBufferStats("COL1");
     stats.addStrValue("abcde|abcde|abcde|abcde|abcde|abcde|");
     Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ab", stats.getCurrentMinColStrValue());
     Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ac", stats.getCurrentMaxColStrValue());
@@ -121,7 +121,7 @@ public void testStrTruncation() throws Exception {
     Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ab", stats.getCurrentMinColStrValue());
     Assert.assertEquals("zabcde|abcde|abcde|abcde|abcde|b", stats.getCurrentMaxColStrValue());
 
-    RowBufferStats ai = new RowBufferStats("en-ai");
+    RowBufferStats ai = new RowBufferStats("COL1", "en-ai");
     ai.addStrValue("abcde|abcde|abcde|abcde|abcde|abcde|");
     Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ab", ai.getCurrentMinColStrValue());
     Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ac", ai.getCurrentMaxColStrValue());
@@ -133,7 +133,7 @@ public void testStrTruncation() throws Exception {
 
   @Test
   public void testMinMaxStrCol() throws Exception {
-    RowBufferStats stats = new RowBufferStats("en-ci");
+    RowBufferStats stats = new RowBufferStats("COL1", "en-ci");
 
     Assert.assertEquals("en-ci", stats.getCollationDefinitionString());
 
@@ -161,7 +161,7 @@ public void testMinMaxStrCol() throws Exception {
 
   @Test
   public void testMinMaxInt() throws Exception {
-    RowBufferStats stats = new RowBufferStats();
+    RowBufferStats stats = new RowBufferStats("COL1");
 
     stats.addIntValue(BigInteger.valueOf(5));
     Assert.assertEquals(BigInteger.valueOf((5)), stats.getCurrentMinIntValue());
@@ -188,7 +188,7 @@ public void testMinMaxInt() throws Exception {
 
   @Test
   public void testMinMaxReal() throws Exception {
-    RowBufferStats stats = new RowBufferStats();
+    RowBufferStats stats = new RowBufferStats("COL1");
 
     stats.addRealValue(1.0);
     Assert.assertEquals(Double.valueOf(1), stats.getCurrentMinRealValue());
@@ -215,7 +215,7 @@ public void testMinMaxReal() throws Exception {
 
   @Test
   public void testIncCurrentNullCount() throws Exception {
-    RowBufferStats stats = new RowBufferStats();
+    RowBufferStats stats = new RowBufferStats("COL1");
 
     Assert.assertEquals(0, stats.getCurrentNullCount());
     stats.incCurrentNullCount();
@@ -226,7 +226,7 @@ public void testIncCurrentNullCount() throws Exception {
 
   @Test
   public void testMaxLength() throws Exception {
-    RowBufferStats stats = new RowBufferStats();
+    RowBufferStats stats = new RowBufferStats("COL1");
 
     Assert.assertEquals(0, stats.getCurrentMaxLength());
     stats.setCurrentMaxLength(100L);
@@ -238,8 +238,8 @@ public void testMaxLength() throws Exception {
   @Test
   public void testGetCombinedStats() throws Exception {
     // Test for Integers
-    RowBufferStats one = new RowBufferStats();
-    RowBufferStats two = new RowBufferStats();
+    RowBufferStats one = new RowBufferStats("COL1");
+    RowBufferStats two = new RowBufferStats("COL1");
 
     one.addIntValue(BigInteger.valueOf(2));
     one.addIntValue(BigInteger.valueOf(4));
@@ -265,8 +265,8 @@ public void testGetCombinedStats() throws Exception {
     Assert.assertNull(result.getCurrentMaxRealValue());
 
     // Test for Reals
-    one = new RowBufferStats();
-    two = new RowBufferStats();
+    one = new RowBufferStats("COL1");
+    two = new RowBufferStats("COL1");
 
     one.addRealValue(2d);
     one.addRealValue(4d);
@@ -291,8 +291,8 @@ public void testGetCombinedStats() throws Exception {
     Assert.assertNull(result.getCurrentMaxIntValue());
 
     // Test for Strings without collation
-    one = new RowBufferStats();
-    two = new RowBufferStats();
+    one = new RowBufferStats("COL1");
+    two = new RowBufferStats("COL1");
 
     one.addStrValue("alpha");
     one.addStrValue("d");
@@ -321,8 +321,8 @@ public void testGetCombinedStats() throws Exception {
     Assert.assertNull(result.getCurrentMaxIntValue());
 
     // Test for Strings with collation
-    one = new RowBufferStats("en-ci");
-    two = new RowBufferStats("en-ci");
+    one = new RowBufferStats("COL1", "en-ci");
+    two = new RowBufferStats("COL1", "en-ci");
 
     one.addStrValue("a");
     one.addStrValue("d");
@@ -351,8 +351,8 @@ public void testGetCombinedStats() throws Exception {
   @Test
   public void testGetCombinedStatsNull() throws Exception {
     // Test for Integers
-    RowBufferStats one = new RowBufferStats();
-    RowBufferStats two = new RowBufferStats();
+    RowBufferStats one = new RowBufferStats("COL1");
+    RowBufferStats two = new RowBufferStats("COL1");
 
     one.addIntValue(BigInteger.valueOf(2));
     one.addIntValue(BigInteger.valueOf(4));
@@ -373,7 +373,7 @@ public void testGetCombinedStatsNull() throws Exception {
     Assert.assertNull(result.getCurrentMaxRealValue());
 
     // Test for Reals
-    one = new RowBufferStats();
+    one = new RowBufferStats("COL1");
 
     one.addRealValue(2d);
     one.addRealValue(4d);
@@ -392,8 +392,8 @@ public void testGetCombinedStatsNull() throws Exception {
     Assert.assertNull(result.getCurrentMaxIntValue());
 
     // Test for Strings
-    one = new RowBufferStats();
-    two = new RowBufferStats();
+    one = new RowBufferStats("COL1");
+    two = new RowBufferStats("COL1");
 
     one.addStrValue("alpha");
     one.addStrValue("d");
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 290c6416c..33c912bc0 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -140,6 +140,7 @@ public void buildFieldErrorStates() {
 
     // Fixed LOB
     testCol = new ColumnMetadata();
+    testCol.setName("COL1");
     testCol.setPhysicalType("LOB");
     testCol.setLogicalType("FIXED");
     try {
@@ -151,6 +152,7 @@ public void buildFieldErrorStates() {
 
     // TIMESTAMP_NTZ SB2
     testCol = new ColumnMetadata();
+    testCol.setName("COL1");
     testCol.setPhysicalType("SB2");
     testCol.setLogicalType("TIMESTAMP_NTZ");
     try {
@@ -162,6 +164,7 @@ public void buildFieldErrorStates() {
 
     // TIMESTAMP_TZ SB1
     testCol = new ColumnMetadata();
+    testCol.setName("COL1");
     testCol.setPhysicalType("SB1");
     testCol.setLogicalType("TIMESTAMP_TZ");
     try {
@@ -173,6 +176,7 @@ public void buildFieldErrorStates() {
 
     // TIME SB16
     testCol = new ColumnMetadata();
+    testCol.setName("COL1");
     testCol.setPhysicalType("SB16");
     testCol.setLogicalType("TIME");
     try {
@@ -441,12 +445,12 @@ private void testDoubleQuotesColumnNameHelper(OpenChannelRequest.OnErrorOption o
   public void testBuildEpInfoFromStats() {
     Map<String, RowBufferStats> colStats = new HashMap<>();
 
-    RowBufferStats stats1 = new RowBufferStats();
+    RowBufferStats stats1 = new RowBufferStats("intColumn");
     stats1.addIntValue(BigInteger.valueOf(2));
     stats1.addIntValue(BigInteger.valueOf(10));
     stats1.addIntValue(BigInteger.valueOf(1));
 
-    RowBufferStats stats2 = new RowBufferStats();
+    RowBufferStats stats2 = new RowBufferStats("strColumn");
     stats2.addStrValue("alice");
     stats2.addStrValue("bob");
     stats2.incCurrentNullCount();
@@ -477,11 +481,13 @@ public void testBuildEpInfoFromNullColumnStats() {
     final String realColName = "realCol";
     Map<String, RowBufferStats> colStats = new HashMap<>();
 
-    RowBufferStats stats = new RowBufferStats();
-    stats.incCurrentNullCount();
+    RowBufferStats stats1 = new RowBufferStats(intColName);
+    RowBufferStats stats2 = new RowBufferStats(realColName);
+    stats1.incCurrentNullCount();
+    stats2.incCurrentNullCount();
 
-    colStats.put(intColName, stats);
-    colStats.put(realColName, stats);
+    colStats.put(intColName, stats1);
+    colStats.put(realColName, stats2);
 
     EpInfo result = AbstractRowBuffer.buildEpInfoFromStats(2, colStats);
     Map<String, FileColumnProperties> columnResults = result.getColumnEps();
@@ -525,7 +531,7 @@ private void testE2EHelper(AbstractRowBuffer<?> rowBuffer) {
     InsertValidationResponse response = rowBuffer.insertRows(Collections.singletonList(row1), null);
     Assert.assertFalse(response.hasErrors());
 
-    Assert.assertEquals((byte) 10, rowBuffer.getVectorValueAt("\"colTinyInt\"", 0));
+    Assert.assertEquals((byte) 10, rowBuffer.getVectorValueAt("colTinyInt", 0));
     Assert.assertEquals((byte) 1, rowBuffer.getVectorValueAt("COLTINYINT", 0));
     Assert.assertEquals((short) 2, rowBuffer.getVectorValueAt("COLSMALLINT", 0));
     Assert.assertEquals(3, rowBuffer.getVectorValueAt("COLINT", 0));
@@ -602,11 +608,11 @@ private void testStatsE2EHelper(AbstractRowBuffer<?> rowBuffer) {
     Map<String, RowBufferStats> columnEpStats = result.getColumnEps();
 
     Assert.assertEquals(
-        BigInteger.valueOf(11), columnEpStats.get("\"colTinyInt\"").getCurrentMaxIntValue());
+        BigInteger.valueOf(11), columnEpStats.get("colTinyInt").getCurrentMaxIntValue());
     Assert.assertEquals(
-        BigInteger.valueOf(10), columnEpStats.get("\"colTinyInt\"").getCurrentMinIntValue());
-    Assert.assertEquals(0, columnEpStats.get("\"colTinyInt\"").getCurrentNullCount());
-    Assert.assertEquals(-1, columnEpStats.get("\"colTinyInt\"").getDistinctValues());
+        BigInteger.valueOf(10), columnEpStats.get("colTinyInt").getCurrentMinIntValue());
+    Assert.assertEquals(0, columnEpStats.get("colTinyInt").getCurrentNullCount());
+    Assert.assertEquals(-1, columnEpStats.get("colTinyInt").getDistinctValues());
 
     Assert.assertEquals(
         BigInteger.valueOf(1), columnEpStats.get("COLTINYINT").getCurrentMaxIntValue());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index 65d97d0b2..91c40eb13 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -460,7 +460,7 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
             .build();
 
     Map<String, RowBufferStats> columnEps = new HashMap<>();
-    columnEps.put("column", new RowBufferStats());
+    columnEps.put("column", new RowBufferStats("COL1"));
     EpInfo epInfo = AbstractRowBuffer.buildEpInfoFromStats(1, columnEps);
 
     ChunkMetadata chunkMetadata =
@@ -501,7 +501,7 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
 
   private Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> getRetryBlobMetadata() {
     Map<String, RowBufferStats> columnEps = new HashMap<>();
-    columnEps.put("column", new RowBufferStats());
+    columnEps.put("column", new RowBufferStats("COL1"));
     EpInfo epInfo = AbstractRowBuffer.buildEpInfoFromStats(1, columnEps);
 
     ChannelMetadata channelMetadata1 =
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index c5189bcde..fb2b1f5b5 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -973,6 +973,88 @@ public void testNullValuesOnMultiDataTypes() throws Exception {
     }
   }
 
+  @Test
+  public void testColumnNameQuotes() throws Exception {
+    final String tableName = "T_COLUMN_WITH_QUOTES";
+    final String unquotedColumn = "c1"; // user name <C1>
+    final String quotedColumn = "\"c 2\""; // user name <c 2>
+    final String doubleQuotedColumn = "\"c\"\"a\\ \"\"\\ 3\""; // user name <c"a\ "\ 3>
+    final String escapedSpacesColumn1 =
+        "c\\ 4"; // space is escaped with '\' in Snowflake SQL, user name <C 4>
+    final String escapedSpacesColumn2 =
+        "c\\ 5"; // space is escaped with '\' in Snowflake SQL, user name <C 5>
+    jdbcConnection
+        .createStatement()
+        .execute(
+            String.format(
+                "create or replace table %s.%s.%s (%s NUMBER(38,0), %s NUMBER(38,0), %s"
+                    + " NUMBER(38,0), %s NUMBER(38,0), %s NUMBER(38,0));",
+                testDb,
+                TEST_SCHEMA,
+                tableName,
+                unquotedColumn,
+                quotedColumn,
+                doubleQuotedColumn,
+                escapedSpacesColumn1,
+                escapedSpacesColumn2));
+
+    OpenChannelRequest request1 =
+        OpenChannelRequest.builder("CHANNEL")
+            .setDBName(testDb)
+            .setSchemaName(TEST_SCHEMA)
+            .setTableName(tableName)
+            .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+            .build();
+
+    // Open a streaming ingest channel from the given client
+    SnowflakeStreamingIngestChannel channel1 = client.openChannel(request1);
+    for (int val = 0; val < 10; val++) {
+      Map<String, Object> row = new HashMap<>();
+      row.put(unquotedColumn, val);
+      row.put(quotedColumn, val);
+      row.put(doubleQuotedColumn, val);
+      row.put(escapedSpacesColumn1, val);
+      // space escape '\' is not really required, because Java always has its own outer double
+      // quotes.
+      // hence, it happens to also work w/o the space escape after the internal literal
+      // normalisation.
+      String escapedSpacesColumn2NoSpaceEscape = escapedSpacesColumn2.replace("\\", "");
+      row.put(escapedSpacesColumn2NoSpaceEscape, val);
+      verifyInsertValidationResponse(channel1.insertRow(row, Integer.toString(val)));
+    }
+
+    // Close the channel after insertion
+    channel1.close().get();
+
+    for (int i = 1; i < 15; i++) {
+      if (channel1.getLatestCommittedOffsetToken() != null
+          && channel1.getLatestCommittedOffsetToken().equals("9")) {
+        ResultSet result =
+            jdbcConnection
+                .createStatement()
+                .executeQuery(
+                    String.format(
+                        "select %s, %s, %s, %s, %s from %s.%s.%s order by %s",
+                        unquotedColumn,
+                        quotedColumn,
+                        doubleQuotedColumn,
+                        escapedSpacesColumn1,
+                        escapedSpacesColumn2,
+                        testDb,
+                        TEST_SCHEMA,
+                        tableName,
+                        unquotedColumn));
+        for (int val = 0; val < 10; val++) {
+          result.next();
+          for (int index = 1; index <= 5; index++) Assert.assertEquals(val, result.getInt(index));
+        }
+        return;
+      }
+      Thread.sleep(500);
+    }
+    Assert.fail("Row sequencer not updated before timeout");
+  }
+
   /** Verify the insert validation response and throw the exception if needed */
   private void verifyInsertValidationResponse(InsertValidationResponse response) {
     if (response.hasErrors()) {
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index 31afda1f0..874ce643b 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -132,12 +132,17 @@ protected String createTable(String dataType) throws SQLException {
   }
 
   protected SnowflakeStreamingIngestChannel openChannel(String tableName) {
+    return openChannel(tableName, OpenChannelRequest.OnErrorOption.ABORT);
+  }
+
+  protected SnowflakeStreamingIngestChannel openChannel(
+      String tableName, OpenChannelRequest.OnErrorOption onErrorOption) {
     OpenChannelRequest openChannelRequest =
         OpenChannelRequest.builder("CHANNEL")
             .setDBName(databaseName)
             .setSchemaName(SCHEMA_NAME)
             .setTableName(tableName)
-            .setOnErrorOption(OpenChannelRequest.OnErrorOption.ABORT)
+            .setOnErrorOption(onErrorOption)
             .build();
     return client.openChannel(openChannelRequest);
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/it/ColumnNamesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/it/ColumnNamesIT.java
new file mode 100644
index 000000000..fc2b220f4
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/it/ColumnNamesIT.java
@@ -0,0 +1,231 @@
+package net.snowflake.ingest.streaming.internal.it;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.UUID;
+import net.snowflake.ingest.TestUtils;
+import net.snowflake.ingest.streaming.InsertValidationResponse;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.streaming.internal.datatypes.AbstractDataTypeTest;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.SFException;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ColumnNamesIT extends AbstractDataTypeTest {
+  private static final int INGEST_VALUE = 1;
+
+  public ColumnNamesIT(String name, Constants.BdecVersion bdecVersion) {
+    super(name, bdecVersion);
+  }
+
+  @Test
+  public void testColumnNamesSupport() throws Exception {
+    // Test simple case
+    testColumnNameSupported("FOO", "FOO");
+    testColumnNameSupported("FOO", "FoO");
+    testColumnNameSupported("FOO", "\"FOO\"");
+    testColumnNameUnsupported("FOO", "\"Foo\"");
+
+    // Test quoted identifier
+    testColumnNameSupported("\"FOO\"", "\"FOO\"");
+    testColumnNameSupported("\"FOO\"", "FOO");
+    testColumnNameSupported("\"FOO\"", "Foo");
+    testColumnNameUnsupported("\"FOO\"", "\"Foo\"");
+
+    testColumnNameSupported("\"Foo\"", "\"Foo\"");
+    testColumnNameUnsupported("\"Foo\"", "Foo");
+
+    // Test keyword
+    testColumnNameSupported("\"CReATE\"", "\"CReATE\"");
+    testColumnNameSupported("\"CREATE\"", "CReATE");
+    testColumnNameUnsupported("\"CReATE\"", "\"CREATE\"");
+    testColumnNameUnsupported("\"CReATE\"", "CReATE");
+
+    // Test escaped space
+    testColumnNameSupported("fO\\ O", "fO\\ O");
+    testColumnNameSupported("fO\\ O", "fO\\ o");
+    testColumnNameSupported("fO\\ O", "fO O");
+    testColumnNameSupported("fO\\ O", "fO o");
+    testColumnNameSupported("fO\\ O", "\"FO O\"");
+    testColumnNameUnsupported("fO\\ O", "\"FO\\ O\"");
+
+    // Test double quotes
+    testColumnNameSupported("\"foo\"\"bar\"", "\"foo\"\"bar\"");
+    testColumnNameSupported("\"FOO\"\"BAR\"", "foo\"bar");
+    testColumnNameSupported("\"\"\"\"", "\"");
+    testColumnNameSupported("\"\"\"\"", "\"\"\"\"");
+    testColumnNameUnsupported("\"\"\"\"", "\"\"\"\"\"\"");
+
+    // Test quoted column with spaces
+    testColumnNameSupported("\"FO O\"", "FO O");
+    testColumnNameSupported("\"FO O\"", "\"FO O\"");
+    testColumnNameUnsupported("\"FO O\"", "\"FO\\ O\"");
+  }
+
+  @Test
+  public void testNonstandardTableAndColumnNames() throws Exception {
+    testColumnNameSupported("fo\\ o");
+    testColumnNameSupported("foo");
+    testColumnNameSupported("\"foo\"");
+    testColumnNameSupported("\"fo o\"");
+    testColumnNameSupported("\"alter\"");
+    testColumnNameSupported("\"table\"");
+    testColumnNameSupported("\"  \"");
+    testColumnNameSupported("\"\"");
+    testColumnNameSupported("\"a\"\"b\"");
+    testColumnNameSupported("\"\"\"\"");
+    testColumnNameSupported("\"  \"\"  \"\"  \"");
+    testColumnNameSupported("\"  \"\" Ť \"\"  \"");
+  }
+
+  /** Tests that quoted columns are correctly resolved for null-backfill */
+  @Test
+  public void testNullableResolution() throws Exception {
+    String tableName = "t1";
+    conn.createStatement()
+        .execute(
+            String.format(
+                "create or replace table %s (AbC int, \"AbC\" int, \"abC\" int, ab\\ c int, \"Ab"
+                    + " c\" int);",
+                tableName));
+    SnowflakeStreamingIngestChannel channel = openChannel(tableName);
+    String offsetToken = "token1";
+    channel.insertRow(new HashMap<>(), offsetToken);
+    TestUtils.waitForOffset(channel, offsetToken);
+
+    ResultSet rs =
+        conn.createStatement().executeQuery(String.format("select * from %s", tableName));
+    rs.next();
+    Assert.assertNull(rs.getObject(1));
+    Assert.assertNull(rs.getObject(2));
+    Assert.assertNull(rs.getObject(3));
+    Assert.assertNull(rs.getObject(4));
+    Assert.assertNull(rs.getObject(5));
+  }
+
+  /**
+   * Test that original user input is used in extra column names validation response (required by
+   * KC)
+   */
+  @Test
+  public void testExtraColNames() throws Exception {
+    String tableName = "t1";
+    conn.createStatement()
+        .execute(String.format("create or replace table %s (\"create\" int);", tableName));
+    SnowflakeStreamingIngestChannel channel =
+        openChannel(tableName, OpenChannelRequest.OnErrorOption.CONTINUE);
+
+    // Test simple input
+    Map<String, Object> row = new HashMap<>();
+    row.put("\"create\"", 4);
+    row.put("abc", 11);
+    InsertValidationResponse insertValidationResponse =
+        channel.insertRow(row, UUID.randomUUID().toString());
+    Assert.assertEquals(1, insertValidationResponse.getInsertErrors().size());
+    Assert.assertEquals(
+        Collections.singletonList("abc"),
+        insertValidationResponse.getInsertErrors().get(0).getExtraColNames());
+
+    // Test quoted input
+    row = new HashMap<>();
+    row.put("\"create\"", 4);
+    row.put("\"CrEaTe\"", 11);
+    insertValidationResponse = channel.insertRow(row, UUID.randomUUID().toString());
+    Assert.assertEquals(1, insertValidationResponse.getInsertErrors().size());
+    Assert.assertEquals(
+        Collections.singletonList("\"CrEaTe\""),
+        insertValidationResponse.getInsertErrors().get(0).getExtraColNames());
+  }
+
+  /** Test that display names are shown in missing not null columns validation response */
+  @Test
+  public void testMissingNotNullColNames() throws Exception {
+    String tableName = "t1";
+    conn.createStatement()
+        .execute(
+            String.format(
+                "create or replace table %s (\"CrEaTe\" int not null, a int not null, \"a\" int not"
+                    + " null, \"create\" int);",
+                tableName));
+    SnowflakeStreamingIngestChannel channel =
+        openChannel(tableName, OpenChannelRequest.OnErrorOption.CONTINUE);
+
+    InsertValidationResponse insertValidationResponse =
+        channel.insertRow(new HashMap<>(), UUID.randomUUID().toString());
+    Assert.assertEquals(1, insertValidationResponse.getInsertErrors().size());
+    Assert.assertEquals(
+        new HashSet<>(Arrays.asList("\"CrEaTe\"", "A", "\"a\"")),
+        new HashSet<>(
+            insertValidationResponse.getInsertErrors().get(0).getMissingNotNullColNames()));
+  }
+
+  /**
+   * Tests that data can be ingested for specific row key into a specific column
+   *
+   * @param createTableColumnName Column name used in CREATE TABLE
+   * @param ingestColumnName Column name of used in ingestion
+   */
+  private void testColumnNameSupported(String createTableColumnName, String ingestColumnName)
+      throws SQLException, InterruptedException {
+
+    String tableName = createSimpleTable(createTableColumnName);
+    String offsetToken = UUID.randomUUID().toString();
+    SnowflakeStreamingIngestChannel channel = openChannel(tableName);
+    Map<String, Object> row = new HashMap<>();
+    row.put(ingestColumnName, INGEST_VALUE);
+    channel.insertRow(row, offsetToken);
+
+    TestUtils.waitForOffset(channel, offsetToken);
+
+    // Verify that supported columns work
+    ResultSet rs =
+        conn.createStatement().executeQuery(String.format("select * from %s;", tableName));
+    int count = 0;
+    while (rs.next()) {
+      count++;
+      Assert.assertEquals(INGEST_VALUE, rs.getInt(1));
+    }
+    Assert.assertEquals(1, count);
+
+    conn.createStatement().execute(String.format("alter table %s migrate;", tableName));
+  }
+
+  private void testColumnNameSupported(String column) throws SQLException, InterruptedException {
+    testColumnNameSupported(column, column);
+  }
+
+  /** Verifies that column names that are not support to work, does not work */
+  private void testColumnNameUnsupported(String createTableColumnName, String ingestColumnName)
+      throws SQLException {
+
+    Map<String, Object> row = new HashMap<>();
+    row.put(ingestColumnName, INGEST_VALUE);
+    SnowflakeStreamingIngestChannel channel = openChannel(createSimpleTable(createTableColumnName));
+    testInsertRowFails(channel, row);
+  }
+
+  private void testInsertRowFails(
+      SnowflakeStreamingIngestChannel channel, Map<String, Object> row) {
+    try {
+      channel.insertRow(row, UUID.randomUUID().toString());
+      Assert.fail("Ingest row should not succeed");
+    } catch (SFException e) {
+      // all good, expected exception has been thrown
+    }
+  }
+
+  private String createSimpleTable(String createTableColumnName) throws SQLException {
+    String tableName = "a" + UUID.randomUUID().toString().replace("-", "_");
+    String createTableSql =
+        String.format("create table %s (%s int);", tableName, createTableColumnName);
+    conn.createStatement().execute(createTableSql);
+    return tableName;
+  }
+}

From d28ef2d1f48ae612077efd961f785d974b25a93f Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Mon, 5 Dec 2022 13:22:38 +0100
Subject: [PATCH 098/356] SNOW-703928 add two system properties for IT tests
 (#296)

SNOW-703928 add two system properties for IT tests
---
 .../java/net/snowflake/ingest/TestUtils.java  | 34 ++++++++++++++++++-
 .../streaming/internal/StreamingIngestIT.java |  8 +----
 .../datatypes/AbstractDataTypeTest.java       |  9 +----
 3 files changed, 35 insertions(+), 16 deletions(-)

diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 9f586a04c..96d46b393 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -28,6 +28,8 @@
 import java.sql.DriverManager;
 import java.sql.ResultSet;
 import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Optional;
 import java.util.Properties;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
@@ -91,7 +93,8 @@ public class TestUtils {
    * @throws IOException if can't read profile
    */
   private static void init() throws Exception {
-    Path path = Paths.get(PROFILE_PATH);
+    String testProfilePath = getTestProfilePath();
+    Path path = Paths.get(testProfilePath);
 
     if (Files.exists(path)) {
       profile = (ObjectNode) mapper.readTree(new String(Files.readAllBytes(path)));
@@ -129,6 +132,35 @@ private static void init() throws Exception {
     }
   }
 
+  /** @return profile path that will be used for tests. */
+  private static String getTestProfilePath() {
+    String testProfilePath =
+        System.getProperty("testProfilePath") != null
+            ? System.getProperty("testProfilePath")
+            : PROFILE_PATH;
+    return testProfilePath;
+  }
+
+  /** @return list of Bdec versions for which to execute IT tests. */
+  public static Collection<Object[]> getBdecVersionItCases() {
+    boolean enableParquetTests =
+        System.getProperty("enableParquetTests") != null
+            && Boolean.parseBoolean(System.getProperty("enableParquetTests"));
+    if (enableParquetTests) {
+      return Arrays.asList(
+          new Object[][] {
+            {"Arrow", Constants.BdecVersion.ONE}, {"Parquet", Constants.BdecVersion.THREE}
+          });
+    }
+    return Arrays.asList(
+        new Object[][] {
+          {"Arrow", Constants.BdecVersion.ONE},
+          // TODO: uncomment once SNOW-659721 is deployed and we set the parameter
+          // DISABLE_PARQUET_CACHE to true for the test account
+          // {"Parquet", Constants.BdecVersion.THREE}
+        });
+  }
+
   public static String getUser() throws Exception {
     if (profile == null) {
       init();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index fb2b1f5b5..b8ceee912 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -48,13 +48,7 @@
 public class StreamingIngestIT {
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> bdecVersion() {
-    return Arrays.asList(
-        new Object[][] {
-          {"Arrow", Constants.BdecVersion.ONE},
-          // TODO: uncomment once SNOW-659721 is deployed and we set the parameter
-          // DISABLE_PARQUET_CACHE to true for the test account
-          // {"Parquet", Constants.BdecVersion.THREE}
-        });
+    return TestUtils.getBdecVersionItCases();
   }
 
   private static final String TEST_TABLE = "STREAMING_INGEST_TEST_TABLE";
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index 874ce643b..d70f449e1 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -9,7 +9,6 @@
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
@@ -33,13 +32,7 @@
 public abstract class AbstractDataTypeTest {
   @Parameterized.Parameters(name = "{0}")
   public static Collection<Object[]> bdecVersion() {
-    return Arrays.asList(
-        new Object[][] {
-          {"Arrow", Constants.BdecVersion.ONE},
-          // TODO: uncomment once SNOW-659721 is deployed and we set the parameter
-          // DISABLE_PARQUET_CACHE to true for the test account
-          // {"Parquet", Constants.BdecVersion.THREE}
-        });
+    return TestUtils.getBdecVersionItCases();
   }
 
   private static final String SOURCE_COLUMN_NAME = "source";

From 6e68e54142f99329fbfead40faa8054dd7a3eeb8 Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Tue, 6 Dec 2022 12:41:45 +0000
Subject: [PATCH 099/356] @snow SNOW-705508 Exclude slf4j-reload4j from Hadoop
 dep

---
 pom.xml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/pom.xml b/pom.xml
index da16b3ee4..7dfa61d6f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -335,6 +335,10 @@
                     <groupId>io.netty</groupId>
                     <artifactId>netty</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-reload4j</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -406,6 +410,10 @@
                     <groupId>org.apache.avro</groupId>
                     <artifactId>avro</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-reload4j</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 

From b6683fab70fdd91ce3d3e8a5bf976729440266ef Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Tue, 6 Dec 2022 11:09:53 -0800
Subject: [PATCH 100/356] NO-SNOW: fix client telemetry logic (#297)

This PR contains two changes on the client telemetry:

1. Only send the telemetry when the count is NOT 0
2. Update the latency unit to milliseconds
---
 .../ingest/connection/TelemetryService.java   | 48 +++++++++++--------
 1 file changed, 28 insertions(+), 20 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java
index 31105c189..ca0a9c41a 100644
--- a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java
+++ b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java
@@ -74,12 +74,14 @@ public void close() {
   /** Report the Streaming Ingest latency metrics */
   public void reportLatencyInSec(
       Timer buildLatency, Timer uploadLatency, Timer registerLatency, Timer flushLatency) {
-    ObjectNode msg = MAPPER.createObjectNode();
-    msg.set("build_latency_sec", buildMsgFromTimer(buildLatency));
-    msg.set("upload_latency_sec", buildMsgFromTimer(uploadLatency));
-    msg.set("register_latency_sec", buildMsgFromTimer(registerLatency));
-    msg.set("flush_latency_sec", buildMsgFromTimer(flushLatency));
-    send(TelemetryType.STREAMING_INGEST_LATENCY_IN_SEC, msg);
+    if (flushLatency.getCount() > 0) {
+      ObjectNode msg = MAPPER.createObjectNode();
+      msg.set("build_latency_ms", buildMsgFromTimer(buildLatency));
+      msg.set("upload_latency_ms", buildMsgFromTimer(uploadLatency));
+      msg.set("register_latency_ms", buildMsgFromTimer(registerLatency));
+      msg.set("flush_latency_ms", buildMsgFromTimer(flushLatency));
+      send(TelemetryType.STREAMING_INGEST_LATENCY_IN_SEC, msg);
+    }
   }
 
   /** Report the Streaming Ingest failure metrics */
@@ -91,24 +93,30 @@ public void reportClientFailure(String summary, String exception) {
   }
 
   /** Report the Streaming Ingest throughput metrics */
-  public void reportThroughputBytesPerSecond(Meter inputThroughput, Meter uploadThrough) {
-    ObjectNode msg = MAPPER.createObjectNode();
-    msg.put("input_mean_rate_bytes_per_sec", inputThroughput.getMeanRate());
-    msg.put("upload_mean_rate_bytes_per_sec", uploadThrough.getMeanRate());
-    send(TelemetryType.STREAMING_INGEST_THROUGHPUT_BYTES_PER_SEC, msg);
+  public void reportThroughputBytesPerSecond(Meter inputThroughput, Meter uploadThroughput) {
+    if (inputThroughput.getCount() > 0) {
+      ObjectNode msg = MAPPER.createObjectNode();
+      msg.put(COUNT, inputThroughput.getCount());
+      msg.put("input_mean_rate_bytes_per_sec", inputThroughput.getMeanRate());
+      msg.put("upload_mean_rate_bytes_per_sec", uploadThroughput.getMeanRate());
+      send(TelemetryType.STREAMING_INGEST_THROUGHPUT_BYTES_PER_SEC, msg);
+    }
   }
 
   /** Report the Streaming Ingest CUP/memory usage metrics */
   public void reportCpuMemoryUsage(Histogram cpuUsage) {
-    ObjectNode msg = MAPPER.createObjectNode();
-    Snapshot cpuSnapshot = cpuUsage.getSnapshot();
-    Runtime runTime = Runtime.getRuntime();
-    msg.put("cpu_max", cpuSnapshot.getMax());
-    msg.put("cpu_mean", cpuSnapshot.getMean());
-    msg.put("max_memory", runTime.maxMemory());
-    msg.put("total_memory", runTime.totalMemory());
-    msg.put("free_memory", runTime.freeMemory());
-    send(TelemetryType.STREAMING_INGEST_CPU_MEMORY_USAGE, msg);
+    if (cpuUsage.getCount() > 0) {
+      ObjectNode msg = MAPPER.createObjectNode();
+      Snapshot cpuSnapshot = cpuUsage.getSnapshot();
+      Runtime runTime = Runtime.getRuntime();
+      msg.put(COUNT, cpuUsage.getCount());
+      msg.put("cpu_max", cpuSnapshot.getMax());
+      msg.put("cpu_mean", cpuSnapshot.getMean());
+      msg.put("max_memory", runTime.maxMemory());
+      msg.put("total_memory", runTime.totalMemory());
+      msg.put("free_memory", runTime.freeMemory());
+      send(TelemetryType.STREAMING_INGEST_CPU_MEMORY_USAGE, msg);
+    }
   }
 
   /** Send log to Snowflake asynchronously through JDBC client telemetry */

From 8549f3dcddc92c7a5273bf58269eea7f2c83eb10 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Wed, 7 Dec 2022 16:33:51 +0100
Subject: [PATCH 101/356] SNOW-706682 Log retry-related messages as INFO (#302)

429 responses are going to occasionally happen, the SDK will retry and
there is no need to scare the users by logging these events on WARN level.
---
 src/main/java/net/snowflake/ingest/utils/HttpUtil.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index 602ca3b6c..3b34459f5 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -231,7 +231,7 @@ public boolean retryRequest(
                 || statusCode >= SERVER_ERRORS);
         if (needNextRetry) {
           long interval = getRetryInterval();
-          LOGGER.warn(
+          LOGGER.info(
               "In retryRequest for service unavailability with statusCode:{} and uri:{}",
               statusCode,
               getRequestUriFromContext(context));

From c8c67e69272de830637705b9eb472d252b2fe105 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 7 Dec 2022 16:51:22 -0800
Subject: [PATCH 102/356] SNOW-705594: fix ConcurrentModificationException on
 the telemetry hashmap (#303)

This PR contains two changes:
1. Fix the ConcurrentModificationException on the telemetry hashmap, use ConcurrentHashMap instead
2. Update the logging for SDK version to INFO from DEBUG
---
 .../net/snowflake/ingest/connection/RequestBuilder.java   | 8 +++++---
 .../snowflake/ingest/streaming/internal/FlushService.java | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 2b438a8dd..b428d23eb 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -313,7 +313,7 @@ private static Properties loadProperties() {
       return properties;
     }
 
-    LOGGER.debug("Loaded project version " + properties.getProperty("version"));
+    LOGGER.info("Loaded project version " + properties.getProperty("version"));
     return properties;
   }
 
@@ -345,9 +345,11 @@ private static String getDefaultUserAgent() {
 
     // Add Java Version
     final String javaVersion = System.getProperty("java.version");
-    defaultUserAgent.append(JAVA_USER_AGENT + "/" + javaVersion);
+    defaultUserAgent.append(JAVA_USER_AGENT + "/").append(javaVersion);
+    String userAgent = defaultUserAgent.toString();
 
-    return defaultUserAgent.toString();
+    LOGGER.info("Default user agent " + userAgent);
+    return userAgent;
   }
 
   private static String buildCustomUserAgent(String additionalUserAgentInfo) {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index a6a80f275..7d75a34d3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -141,7 +141,7 @@ List<List<ChannelData<T>>> getData() {
     this.isNeedFlush = false;
     this.lastFlushTime = System.currentTimeMillis();
     this.isTestMode = isTestMode;
-    this.latencyTimerContextMap = new HashMap<>();
+    this.latencyTimerContextMap = new ConcurrentHashMap<>();
     this.bdecVersion = this.owningClient.getParameterProvider().getBlobFormatVersion();
     createWorkers();
   }

From dd15d1e8e32b5dd3a9b7ddd01940fa821958a6af Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 8 Dec 2022 18:15:08 +0000
Subject: [PATCH 103/356] beta.7 release

---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 7dfa61d6f..45afd3932 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
     <!-- Arifact name and version information -->
     <groupId>net.snowflake</groupId>
     <artifactId>snowflake-ingest-sdk</artifactId>
-    <version>1.0.2-beta.6</version>
+    <version>1.0.2-beta.7</version>
     <packaging>jar</packaging>
     <name>Snowflake Ingest SDK</name>
     <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index b428d23eb..bdb3e0321 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -127,7 +127,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.0.2-beta.6";
+  public static final String DEFAULT_VERSION = "1.0.2-beta.7";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 062387912977e915562f24d870b4adc270afabac Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Fri, 9 Dec 2022 16:51:54 +0100
Subject: [PATCH 104/356] SNOW-707520 Fix incorrect fractional part calculation
 of TIMESTAMP (#304)

When TIMESTAMP columns scale is less than 9, fractional part in Arrow
is stored with less than 9 digits, which is causing mismatch during
scanback. Arrow  expects 9 digits for the fractional part.
---
 .../streaming/internal/DataValidationUtil.java      | 13 +++++++------
 .../streaming/internal/DataValidationUtilTest.java  |  2 +-
 .../internal/datatypes/AbstractDataTypeTest.java    |  6 ++++++
 .../streaming/internal/datatypes/StringsIT.java     |  1 +
 4 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index de7185009..5f3af68d0 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -339,12 +339,13 @@ else if (input instanceof OffsetDateTime)
             valueString, effectiveTimeZone, 0, DATE | TIMESTAMP, ignoreTimezone, null);
     if (timestamp != null) {
       long epoch = timestamp.getSeconds().longValue();
-      int fraction = getFractionFromTimestamp(timestamp) / Power10.intTable[9 - scale];
+      int fractionInScale = getFractionFromTimestamp(timestamp) / Power10.intTable[9 - scale];
       BigInteger timeInScale =
           BigInteger.valueOf(epoch)
               .multiply(Power10.sb16Table[scale])
-              .add(BigInteger.valueOf(fraction));
-      return new TimestampWrapper(epoch, fraction, timeInScale);
+              .add(BigInteger.valueOf(fractionInScale));
+      return new TimestampWrapper(
+          epoch, fractionInScale * Power10.intTable[9 - scale], timeInScale);
     } else {
       throw valueFormatNotAllowedException(
           input.toString(),
@@ -408,11 +409,11 @@ else if (input instanceof OffsetDateTime)
             stringInput, DEFAULT_TIMEZONE, 0, DATE | TIMESTAMP, false, null);
     if (timestamp != null) {
       long epoch = timestamp.getSeconds().longValue();
-      int fraction = getFractionFromTimestamp(timestamp) / Power10.intTable[9 - scale];
+      int fractionInScale = getFractionFromTimestamp(timestamp) / Power10.intTable[9 - scale];
       return new TimestampWrapper(
           epoch,
-          fraction,
-          BigInteger.valueOf(epoch * Power10.intTable[scale] + fraction),
+          fractionInScale * Power10.intTable[9 - scale],
+          BigInteger.valueOf(epoch * Power10.intTable[scale] + fractionInScale),
           timestamp);
     } else {
       throw valueFormatNotAllowedException(
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 2d36de72c..148021016 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -229,7 +229,7 @@ public void testValidateAndPareTimestampTz() {
     TimestampWrapper result =
         DataValidationUtil.validateAndParseTimestampTz("2021-01-01 01:00:00.123 +0100", 4);
     assertEquals(1609459200, result.getEpoch());
-    assertEquals(1230, result.getFraction());
+    assertEquals(123000000, result.getFraction());
     assertEquals(Optional.of(3600000), result.getTimezoneOffset());
     assertEquals(Optional.of(1500), result.getTimeZoneIndex());
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index d70f449e1..ae453da31 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -304,6 +304,7 @@ <STREAMING_INGEST_WRITE, JDBC_WRITE, JDBC_READ> void ingestAndAssert(
     Assert.assertTrue(resultSet.next());
     int count = resultSet.getInt(1);
     Assert.assertEquals(insertAlsoWithJdbc ? 2 : 1, count);
+    migrateTable(tableName); // migration should always succeed
   }
 
   <STREAMING_INGEST_WRITE> void assertVariant(
@@ -337,5 +338,10 @@ <STREAMING_INGEST_WRITE> void assertVariant(
     Assert.assertEquals(1, counter);
     Assert.assertEquals(objectMapper.readTree(expectedValue), objectMapper.readTree(value));
     Assert.assertEquals(expectedType, typeof);
+    migrateTable(tableName); // migration should always succeed
+  }
+
+  protected void migrateTable(String tableName) throws SQLException {
+    conn.createStatement().execute(String.format("alter table %s migrate;", tableName));
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
index 198d1c99c..95681863f 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
@@ -62,6 +62,7 @@ public void testStrings() throws Exception {
   }
 
   @Test
+  @Ignore("SNOW-663621")
   public void testNonAsciiStrings() throws Exception {
     testIngestion(
         "VARCHAR", "ž, š, č, ř, c, j, ď, ť, ň", "ž, š, č, ř, c, j, ď, ť, ň", new StringProvider());

From b0afef76ddb134c52d76306fb300c3071a06fc73 Mon Sep 17 00:00:00 2001
From: Konstantinos Kloudas <konstantinos.kloudas@snowflake.com>
Date: Mon, 5 Dec 2022 15:01:04 +0100
Subject: [PATCH 105/356] SNOW-704566 Add filename in Parquet extra MD

---
 .../streaming/internal/AbstractRowBuffer.java | 13 ++++---
 .../streaming/internal/ArrowRowBuffer.java    |  2 +-
 .../streaming/internal/FlushService.java      | 11 ++++--
 .../streaming/internal/ParquetRowBuffer.java  |  9 ++++-
 .../ingest/streaming/internal/RowBuffer.java  |  3 +-
 ...owflakeStreamingIngestChannelInternal.java |  5 +--
 .../internal/StreamingIngestStage.java        | 18 +++++++++-
 .../internal/StreamingIngestUtils.java        |  5 +++
 .../net/snowflake/ingest/utils/Constants.java |  2 ++
 .../streaming/internal/ArrowBufferTest.java   |  3 +-
 .../streaming/internal/FlushServiceTest.java  |  2 +-
 .../streaming/internal/RowBufferTest.java     | 34 +++++++++++++------
 .../SnowflakeStreamingIngestChannelTest.java  |  4 +--
 .../internal/StreamingIngestStageTest.java    | 13 ++++++-
 14 files changed, 96 insertions(+), 28 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 0a61aa058..4f993534e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -343,10 +343,11 @@ public InsertValidationResponse insertRows(
    * Flush the data in the row buffer by taking the ownership of the old vectors and pass all the
    * required info back to the flush service to build the blob
    *
+   * @param filePath the name of the file the data will be written in
    * @return A ChannelData object that contains the info needed by the flush service to build a blob
    */
   @Override
-  public ChannelData<T> flush() {
+  public ChannelData<T> flush(final String filePath) {
     logger.logDebug("Start get data for channel={}", channelFullyQualifiedName);
     if (this.rowCount > 0) {
       Optional<T> oldData = Optional.empty();
@@ -363,7 +364,7 @@ public ChannelData<T> flush() {
       try {
         if (this.rowCount > 0) {
           // Transfer the ownership of the vectors
-          oldData = getSnapshot();
+          oldData = getSnapshot(filePath);
           oldRowCount = this.rowCount;
           oldBufferSize = this.bufferSize;
           oldRowSequencer = this.channelState.incrementAndGetRowSequencer();
@@ -451,8 +452,12 @@ void reset() {
             new RowBufferStats(value.getColumnDisplayName(), value.getCollationDefinitionString()));
   }
 
-  /** Get buffered data snapshot for later flushing. */
-  abstract Optional<T> getSnapshot();
+  /**
+   * Get buffered data snapshot for later flushing.
+   *
+   * @param filePath the name of the file the data will be written in
+   */
+  abstract Optional<T> getSnapshot(final String filePath);
 
   @VisibleForTesting
   abstract Object getVectorValueAt(String column, int index);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 6776b310c..52d810f05 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -366,7 +366,7 @@ boolean hasColumns() {
   }
 
   @Override
-  Optional<VectorSchemaRoot> getSnapshot() {
+  Optional<VectorSchemaRoot> getSnapshot(final String filePath) {
     List<FieldVector> oldVectors = new ArrayList<>();
     for (FieldVector vector : this.vectorsRoot.getFieldVectors()) {
       vector.setValueCount(this.rowCount);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 7d75a34d3..93245eb9d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -341,6 +341,8 @@ void distributeFlushTasks() {
       List<List<ChannelData<T>>> blobData = new ArrayList<>();
       float totalBufferSize = 0;
 
+      final String filePath = getFilePath(this.targetStage.getClientPrefix());
+
       // Distribute work at table level, create a new blob if reaching the blob size limit
       while (itr.hasNext() && totalBufferSize <= MAX_BLOB_SIZE_IN_BYTES) {
         ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<T>> table =
@@ -349,7 +351,7 @@ void distributeFlushTasks() {
         // TODO: we could do parallel stream to get the channelData if needed
         for (SnowflakeStreamingIngestChannelInternal<T> channel : table.values()) {
           if (channel.isValid()) {
-            ChannelData<T> data = channel.getData();
+            ChannelData<T> data = channel.getData(filePath);
             if (data != null) {
               channelsDataPerTable.add(data);
               totalBufferSize += data.getBufferSize();
@@ -362,8 +364,11 @@ void distributeFlushTasks() {
       }
 
       // Kick off a build job
-      if (!blobData.isEmpty()) {
-        String filePath = getFilePath(this.targetStage.getClientPrefix());
+      if (blobData.isEmpty()) {
+        // we decrement the counter so that we do not have gaps in the filenames created by this
+        // client. See method getFilePath() below.
+        this.counter.decrementAndGet();
+      } else {
         if (this.owningClient.flushLatency != null) {
           latencyTimerContextMap.putIfAbsent(filePath, this.owningClient.flushLatency.time());
         }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 1e918e061..9943c349e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -17,6 +17,7 @@
 import java.util.function.Consumer;
 import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.Pair;
 import org.apache.arrow.memory.BufferAllocator;
@@ -165,9 +166,15 @@ boolean hasColumns() {
   }
 
   @Override
-  Optional<ParquetChunkData> getSnapshot() {
+  Optional<ParquetChunkData> getSnapshot(final String filePath) {
     List<List<Object>> oldData = new ArrayList<>();
     data.forEach(r -> oldData.add(new ArrayList<>(r)));
+
+    // We insert the filename in the file itself as metadata so that streams can work on replicated
+    // mixed tables. For a more detailed discussion on the topic see SNOW-561447 and
+    // http://go/streams-on-replicated-mixed-tables
+    metadata.put(Constants.PRIMARY_FILE_ID_KEY, StreamingIngestUtils.getShortname(filePath));
+
     return oldData.isEmpty()
         ? Optional.empty()
         : Optional.of(new ParquetChunkData(oldData, metadata));
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
index c32720c42..68c225cd4 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
@@ -36,9 +36,10 @@ interface RowBuffer<T> {
    * Flush the data in the row buffer by taking the ownership of the old vectors and pass all the
    * required info back to the flush service to build the blob
    *
+   * @param filePath the name of the file the data will be written in
    * @return A ChannelData object that contains the info needed by the flush service to build a blob
    */
-  ChannelData<T> flush();
+  ChannelData<T> flush(final String filePath);
 
   /**
    * Close the row buffer and release resources. Note that the caller needs to handle
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 3a38f5e7b..124e1cac6 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -186,10 +186,11 @@ public String getFullyQualifiedTableName() {
   /**
    * Get all the data needed to build the blob during flush
    *
+   * @param filePath the name of the file the data will be written in
    * @return a ChannelData object
    */
-  ChannelData<T> getData() {
-    ChannelData<T> data = this.rowBuffer.flush();
+  ChannelData<T> getData(final String filePath) {
+    ChannelData<T> data = this.rowBuffer.flush(filePath);
     if (data != null) {
       data.setChannelContext(channelFlushContext);
     }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index f9d2eb4a3..c389f3d3e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -226,7 +226,7 @@ synchronized SnowflakeFileTransferMetadataWithAge refreshSnowflakeMetadata(boole
     JsonNode responseNode = this.parseClientConfigureResponse(response);
     // Do not change the prefix everytime we have to refresh credentials
     if (Utils.isNullOrEmpty(this.clientPrefix)) {
-      this.clientPrefix = responseNode.get("prefix").textValue();
+      this.clientPrefix = createClientPrefix(responseNode);
     }
     Utils.assertStringNotNullOrEmpty("client prefix", this.clientPrefix);
 
@@ -259,6 +259,22 @@ synchronized SnowflakeFileTransferMetadataWithAge refreshSnowflakeMetadata(boole
     return this.fileTransferMetadataWithAge;
   }
 
+  /**
+   * Creates a client-specific prefix that will be also part of the files registered by this client.
+   * The prefix will include a server-side generated string and the GlobalID of the deployment the
+   * client is registering blobs to. The latter (deploymentId) is needed in order to guarantee that
+   * blob filenames are unique across deployments even with replication enabled.
+   *
+   * @param response the client/configure response from the server
+   * @return the client prefix.
+   */
+  private String createClientPrefix(final JsonNode response) {
+    final String prefix = response.get("prefix").textValue();
+    final String deploymentId =
+        response.has("deployment_id") ? "_" + response.get("deployment_id").longValue() : "";
+    return prefix + deploymentId;
+  }
+
   /**
    * GCS requires a signed url per file. We need to fetch this from the server for each put
    *
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
index db6ac4e61..88b406a62 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
@@ -117,4 +117,9 @@ static <T> T executeWithRetries(
     } while (retries <= MAX_STREAMING_INGEST_API_CHANNEL_RETRY);
     return response;
   }
+
+  public static String getShortname(final String fullname) {
+    final String[] parts = fullname.split("/");
+    return parts[parts.length - 1];
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index 6a479a655..1440851b9 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -25,6 +25,8 @@ public class Constants {
   public static final String PRIVATE_KEY = "private_key";
   public static final String PRIVATE_KEY_PASSPHRASE = "private_key_passphrase";
   public static final String JDBC_PRIVATE_KEY = "privateKey";
+  public static final String PRIMARY_FILE_ID_KEY =
+      "primaryFileId"; // Don't change, should match Parquet Scanner
   public static final long RESPONSE_SUCCESS = 0L; // Don't change, should match server side
   public static final long RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST =
       10L; // Don't change, should match server side
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
index 167e0d1b1..08e559bcc 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
@@ -48,7 +48,8 @@ public void testFieldNumberAfterFlush() {
         rowBufferOnErrorContinue.insertRows(Collections.singletonList(row), offsetToken);
     Assert.assertFalse(response.hasErrors());
 
-    ChannelData<VectorSchemaRoot> data = rowBufferOnErrorContinue.flush();
+    ChannelData<VectorSchemaRoot> data =
+        rowBufferOnErrorContinue.flush("my_snowpipe_streaming.bdec");
     Assert.assertEquals(7, data.getVectors().getFieldVectors().size());
   }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index ce9a44d4d..772e7057e 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -101,7 +101,7 @@ private abstract static class TestContext<T> implements AutoCloseable {
 
     ChannelData<T> flushChannel(String name) {
       SnowflakeStreamingIngestChannelInternal<T> channel = channels.get(name);
-      ChannelData<T> channelData = channel.getRowBuffer().flush();
+      ChannelData<T> channelData = channel.getRowBuffer().flush(name + "_snowpipe_streaming.bdec");
       channelData.setChannelContext(channel.getChannelContext());
       this.channelData.add(channelData);
       return channelData;
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 33c912bc0..b5ed831b4 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -407,11 +407,19 @@ private void testFlushHelper(AbstractRowBuffer<?> rowBuffer) {
     Assert.assertFalse(response.hasErrors());
     float bufferSize = rowBuffer.getSize();
 
-    ChannelData<?> data = rowBuffer.flush();
+    final String filename = "2022/7/13/16/56/testFlushHelper_streaming.bdec";
+    ChannelData<?> data = rowBuffer.flush(filename);
     Assert.assertEquals(2, data.getRowCount());
     Assert.assertEquals((Long) 1L, data.getRowSequencer());
     Assert.assertEquals(offsetToken, data.getOffsetToken());
     Assert.assertEquals(bufferSize, data.getBufferSize(), 0);
+
+    if (bdecVersion == Constants.BdecVersion.THREE) {
+      final ParquetChunkData chunkData = (ParquetChunkData) data.getVectors();
+      Assert.assertEquals(
+          StreamingIngestUtils.getShortname(filename),
+          chunkData.metadata.get(Constants.PRIMARY_FILE_ID_KEY));
+    }
   }
 
   @Test
@@ -602,9 +610,10 @@ private void testStatsE2EHelper(AbstractRowBuffer<?> rowBuffer) {
     row2.put("colDecimal", 4);
     row2.put("colChar", "alice");
 
+    final String filename = "testStatsE2EHelper_streaming.bdec";
     InsertValidationResponse response = rowBuffer.insertRows(Arrays.asList(row1, row2), null);
     Assert.assertFalse(response.hasErrors());
-    ChannelData<?> result = rowBuffer.flush();
+    ChannelData<?> result = rowBuffer.flush(filename);
     Map<String, RowBufferStats> columnEpStats = result.getColumnEps();
 
     Assert.assertEquals(
@@ -645,8 +654,13 @@ private void testStatsE2EHelper(AbstractRowBuffer<?> rowBuffer) {
     Assert.assertEquals(0, columnEpStats.get("COLCHAR").getCurrentNullCount());
     Assert.assertEquals(-1, columnEpStats.get("COLCHAR").getDistinctValues());
 
+    if (bdecVersion == Constants.BdecVersion.THREE) {
+      final ParquetChunkData chunkData = (ParquetChunkData) result.getVectors();
+      Assert.assertEquals(filename, chunkData.metadata.get(Constants.PRIMARY_FILE_ID_KEY));
+    }
+
     // Confirm we reset
-    ChannelData<?> resetResults = rowBuffer.flush();
+    ChannelData<?> resetResults = rowBuffer.flush("my_snowpipe_streaming.bdec");
     Assert.assertNull(resetResults);
   }
 
@@ -701,7 +715,7 @@ private void testStatsE2ETimestampHelper(OpenChannelRequest.OnErrorOption onErro
     InsertValidationResponse response =
         innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null);
     Assert.assertFalse(response.hasErrors());
-    ChannelData<?> result = innerBuffer.flush();
+    ChannelData<?> result = innerBuffer.flush("my_snowpipe_streaming.bdec");
     Assert.assertEquals(3, result.getRowCount());
 
     Assert.assertEquals(
@@ -768,7 +782,7 @@ private void testE2EDateHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
     Assert.assertNull(innerBuffer.getVectorValueAt("COLDATE", 2));
 
     // Check stats generation
-    ChannelData<?> result = innerBuffer.flush();
+    ChannelData<?> result = innerBuffer.flush("my_snowpipe_streaming.bdec");
     Assert.assertEquals(3, result.getRowCount());
 
     Assert.assertEquals(
@@ -831,7 +845,7 @@ private void testE2ETimeHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
     Assert.assertNull(innerBuffer.getVectorValueAt("COLTIMESB8", 2));
 
     // Check stats generation
-    ChannelData<?> result = innerBuffer.flush();
+    ChannelData<?> result = innerBuffer.flush("my_snowpipe_streaming.bdec");
     Assert.assertEquals(3, result.getRowCount());
 
     Assert.assertEquals(
@@ -1002,7 +1016,7 @@ private void testE2EBooleanHelper(OpenChannelRequest.OnErrorOption onErrorOption
     Assert.assertNull(innerBuffer.getVectorValueAt("COLBOOLEAN", 2));
 
     // Check stats generation
-    ChannelData<?> result = innerBuffer.flush();
+    ChannelData<?> result = innerBuffer.flush("my_snowpipe_streaming.bdec");
     Assert.assertEquals(3, result.getRowCount());
 
     Assert.assertEquals(
@@ -1054,7 +1068,7 @@ private void testE2EBinaryHelper(OpenChannelRequest.OnErrorOption onErrorOption)
     Assert.assertNull(innerBuffer.getVectorValueAt("COLBINARY", 2));
 
     // Check stats generation
-    ChannelData<?> result = innerBuffer.flush();
+    ChannelData<?> result = innerBuffer.flush("my_snowpipe_streaming.bdec");
 
     Assert.assertEquals(3, result.getRowCount());
     Assert.assertEquals(11L, result.getColumnEps().get("COLBINARY").getCurrentMaxLength());
@@ -1102,7 +1116,7 @@ private void testE2ERealHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
     Assert.assertNull(innerBuffer.getVectorValueAt("COLREAL", 2));
 
     // Check stats generation
-    ChannelData<?> result = innerBuffer.flush();
+    ChannelData<?> result = innerBuffer.flush("my_snowpipe_streaming.bdec");
 
     Assert.assertEquals(3, result.getRowCount());
     Assert.assertEquals(
@@ -1183,7 +1197,7 @@ public void testOnErrorAbortFailures() {
     Assert.assertNull(innerBuffer.tempStatsMap.get("COLDECIMAL").getCurrentMaxIntValue());
     Assert.assertNull(innerBuffer.tempStatsMap.get("COLDECIMAL").getCurrentMinIntValue());
 
-    ChannelData<?> data = innerBuffer.flush();
+    ChannelData<?> data = innerBuffer.flush("my_snowpipe_streaming.bdec");
     Assert.assertEquals(3, data.getRowCount());
     Assert.assertEquals(0, innerBuffer.rowCount);
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 847972376..162304aae 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -494,7 +494,7 @@ public void testInsertRow() {
     row.put("col", 1);
 
     // Get data before insert to verify that there is no row (data should be null)
-    ChannelData<VectorSchemaRoot> data = channel.getData();
+    ChannelData<VectorSchemaRoot> data = channel.getData("my_snowpipe_streaming.bdec");
     Assert.assertNull(data);
 
     InsertValidationResponse response = channel.insertRow(row, "1");
@@ -503,7 +503,7 @@ public void testInsertRow() {
     Assert.assertFalse(response.hasErrors());
 
     // Get data again to verify the row is inserted
-    data = channel.getData();
+    data = channel.getData("my_snowpipe_streaming.bdec");
     Assert.assertEquals(2, data.getRowCount());
     Assert.assertEquals((Long) 1L, data.getRowSequencer());
     Assert.assertEquals(1, data.getVectors().getFieldVectors().size());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
index 500791673..79c809e24 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
@@ -54,6 +54,11 @@
 @RunWith(PowerMockRunner.class)
 @PrepareForTest({TestUtils.class, HttpUtil.class, SnowflakeFileTransferAgent.class})
 public class StreamingIngestStageTest {
+
+  private final String prefix = "EXAMPLE_PREFIX";
+
+  private final long deploymentId = 123;
+
   private ObjectMapper mapper = new ObjectMapper();
 
   final String exampleRemoteMeta =
@@ -78,8 +83,13 @@ public class StreamingIngestStageTest {
 
   String exampleRemoteMetaResponse =
       "{\"src_locations\": [\"foo/\"],"
+          + " \"deployment_id\": "
+          + deploymentId
+          + ","
           + " \"status_code\": 0, \"message\": \"Success\", \"prefix\":"
-          + " \"EXAMPLE_PREFIX\", \"stage_location\": {\"locationType\": \"S3\", \"location\":"
+          + " \""
+          + prefix
+          + "\", \"stage_location\": {\"locationType\": \"S3\", \"location\":"
           + " \"foo/streaming_ingest/\", \"path\": \"streaming_ingest/\", \"region\":"
           + " \"us-east-1\", \"storageAccount\": null, \"isClientSideEncrypted\": true,"
           + " \"creds\": {\"AWS_KEY_ID\": \"EXAMPLE_AWS_KEY_ID\", \"AWS_SECRET_KEY\":"
@@ -279,6 +289,7 @@ public void testRefreshSnowflakeMetadataRemote() throws Exception {
         "foo/streaming_ingest/", metadataWithAge.fileTransferMetadata.getStageInfo().getLocation());
     Assert.assertEquals(
         "placeholder", metadataWithAge.fileTransferMetadata.getPresignedUrlFileName());
+    Assert.assertEquals(prefix + "_" + deploymentId, stage.getClientPrefix());
   }
 
   @Test

From 2b0dd86f48e7bae892d70b3e4fb1006d7cacc6f7 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Thu, 15 Dec 2022 13:56:28 +0100
Subject: [PATCH 106/356] SNOW-710291 Allow inf, -inf and nan strings into
 FLOAT columns (#306)

---
 .../streaming/internal/DataValidationUtil.java    | 15 +++++++++++++--
 .../internal/DataValidationUtilTest.java          |  5 ++++-
 .../internal/datatypes/NumericTypesIT.java        |  8 ++++++++
 3 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 5f3af68d0..61fe83e97 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -638,10 +638,21 @@ static double validateAndParseReal(Object input) {
     } else if (input instanceof Number) {
       return ((Number) input).doubleValue();
     } else if (input instanceof String) {
+      String stringInput = (String) input;
       try {
-        return Double.parseDouble((String) input);
+        return Double.parseDouble(stringInput);
       } catch (NumberFormatException err) {
-        throw valueFormatNotAllowedException(input, "REAL", "Not a valid decimal number");
+        stringInput = stringInput.toLowerCase();
+        switch (stringInput) {
+          case "nan":
+            return Double.NaN;
+          case "inf":
+            return Double.POSITIVE_INFINITY;
+          case "-inf":
+            return Double.NEGATIVE_INFINITY;
+          default:
+            throw valueFormatNotAllowedException(input, "REAL", "Not a valid decimal number");
+        }
       }
     }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 148021016..7b95c69aa 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -792,7 +792,7 @@ public void testValidateAndParseBinary() {
   }
 
   @Test
-  public void testValidateAndParseReal() throws Exception {
+  public void testValidateAndParseReal() {
     // From number types
     assertEquals(1.23d, validateAndParseReal(1.23f), 0);
     assertEquals(1.23d, validateAndParseReal(1.23), 0);
@@ -803,6 +803,9 @@ public void testValidateAndParseReal() throws Exception {
     assertEquals(1.23d, validateAndParseReal("1.23"), 0);
     assertEquals(123d, validateAndParseReal("1.23E2"), 0);
     assertEquals(123d, validateAndParseReal("1.23e2"), 0);
+    assertEquals(Double.NaN, validateAndParseReal("Nan"), 0);
+    assertEquals(Double.POSITIVE_INFINITY, validateAndParseReal("inF"), 0);
+    assertEquals(Double.NEGATIVE_INFINITY, validateAndParseReal("-inF"), 0);
 
     // Test forbidden values
     expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseReal, "foo");
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
index 64c9bc3fc..e9361e4a0 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
@@ -282,6 +282,14 @@ public void testFloatingPointTypes() throws Exception {
     testJdbcTypeCompatibility("REAL", 1, 1., new IntProvider(), new DoubleProvider());
     testJdbcTypeCompatibility("REAL", 1L, 1., new LongProvider(), new DoubleProvider());
     testJdbcTypeCompatibility("REAL", "1.35", 1.35, new StringProvider(), new DoubleProvider());
+
+    testJdbcTypeCompatibility(
+        "REAL", "Nan", Double.NaN, new StringProvider(), new DoubleProvider());
+    testJdbcTypeCompatibility(
+        "REAL", "Inf", Double.POSITIVE_INFINITY, new StringProvider(), new DoubleProvider());
+    testJdbcTypeCompatibility(
+        "REAL", "-Inf", Double.NEGATIVE_INFINITY, new StringProvider(), new DoubleProvider());
+
     testIngestion("REAL", new BigDecimal("1.35"), new BigDecimal("1.35"), new BigDecimalProvider());
     testIngestion("REAL", BigInteger.ONE, BigDecimal.ONE, new BigDecimalProvider());
   }

From 41331b96aab4d770bc32adf55f8cdff9690b5284 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Mon, 28 Nov 2022 10:56:40 +0000
Subject: [PATCH 107/356] SNOW-684267 Add column name to data validation error
 messages

---
 .../streaming/internal/ArrowRowBuffer.java    |  47 +-
 .../internal/DataValidationUtil.java          |  92 +-
 .../internal/ParquetValueParser.java          |  47 +-
 .../internal/DataValidationUtilTest.java      | 787 ++++++++++--------
 4 files changed, 539 insertions(+), 434 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 52d810f05..320558dad 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -466,7 +466,8 @@ private float convertRowToArrow(
           case FIXED:
             int columnPrecision = Integer.parseInt(field.getMetadata().get(COLUMN_PRECISION));
             int columnScale = getColumnScale(field.getMetadata());
-            BigDecimal inputAsBigDecimal = DataValidationUtil.validateAndParseBigDecimal(value);
+            BigDecimal inputAsBigDecimal =
+                DataValidationUtil.validateAndParseBigDecimal(stats.getColumnDisplayName(), value);
             // vector.setSafe requires the BigDecimal input scale explicitly match its scale
             inputAsBigDecimal = inputAsBigDecimal.setScale(columnScale, RoundingMode.HALF_UP);
 
@@ -511,7 +512,9 @@ private float convertRowToArrow(
               String maxLengthString = field.getMetadata().get(COLUMN_CHAR_LENGTH);
               String str =
                   DataValidationUtil.validateAndParseString(
-                      value, Optional.ofNullable(maxLengthString).map(Integer::parseInt));
+                      stats.getColumnDisplayName(),
+                      value,
+                      Optional.ofNullable(maxLengthString).map(Integer::parseInt));
               Text text = new Text(str);
               ((VarCharVector) vector).setSafe(curRowIndex, text);
               stats.addStrValue(str);
@@ -520,7 +523,8 @@ private float convertRowToArrow(
             }
           case OBJECT:
             {
-              String str = DataValidationUtil.validateAndParseObject(value);
+              String str =
+                  DataValidationUtil.validateAndParseObject(stats.getColumnDisplayName(), value);
               Text text = new Text(str);
               ((VarCharVector) vector).setSafe(curRowIndex, text);
               rowBufferSize += text.getBytes().length;
@@ -528,7 +532,8 @@ private float convertRowToArrow(
             }
           case ARRAY:
             {
-              String str = DataValidationUtil.validateAndParseArray(value);
+              String str =
+                  DataValidationUtil.validateAndParseArray(stats.getColumnDisplayName(), value);
               Text text = new Text(str);
               ((VarCharVector) vector).setSafe(curRowIndex, text);
               rowBufferSize += text.getBytes().length;
@@ -536,7 +541,8 @@ private float convertRowToArrow(
             }
           case VARIANT:
             {
-              String str = DataValidationUtil.validateAndParseVariant(value);
+              String str =
+                  DataValidationUtil.validateAndParseVariant(stats.getColumnDisplayName(), value);
               Text text = new Text(str);
               ((VarCharVector) vector).setSafe(curRowIndex, text);
               rowBufferSize += text.getBytes().length;
@@ -552,7 +558,10 @@ private float convertRowToArrow(
                   BigIntVector bigIntVector = (BigIntVector) vector;
                   TimestampWrapper timestampWrapper =
                       DataValidationUtil.validateAndParseTimestampNtzSb16(
-                          value, getColumnScale(field.getMetadata()), ignoreTimezone);
+                          stats.getColumnDisplayName(),
+                          value,
+                          getColumnScale(field.getMetadata()),
+                          ignoreTimezone);
                   bigIntVector.setSafe(curRowIndex, timestampWrapper.getTimeInScale().longValue());
                   stats.addIntValue(timestampWrapper.getTimeInScale());
                   rowBufferSize += 8;
@@ -570,7 +579,10 @@ private float convertRowToArrow(
 
                   TimestampWrapper timestampWrapper =
                       DataValidationUtil.validateAndParseTimestampNtzSb16(
-                          value, getColumnScale(field.getMetadata()), ignoreTimezone);
+                          stats.getColumnDisplayName(),
+                          value,
+                          getColumnScale(field.getMetadata()),
+                          ignoreTimezone);
                   epochVector.setSafe(curRowIndex, timestampWrapper.getEpoch());
                   fractionVector.setSafe(curRowIndex, timestampWrapper.getFraction());
                   rowBufferSize += 12;
@@ -595,7 +607,7 @@ private float convertRowToArrow(
 
                   TimestampWrapper timestampWrapper =
                       DataValidationUtil.validateAndParseTimestampTz(
-                          value, getColumnScale(field.getMetadata()));
+                          stats.getColumnDisplayName(), value, getColumnScale(field.getMetadata()));
                   epochVector.setSafe(curRowIndex, timestampWrapper.getTimeInScale().longValue());
                   timezoneVector.setSafe(
                       curRowIndex,
@@ -635,7 +647,7 @@ private float convertRowToArrow(
 
                   TimestampWrapper timestampWrapper =
                       DataValidationUtil.validateAndParseTimestampTz(
-                          value, getColumnScale(field.getMetadata()));
+                          stats.getColumnDisplayName(), value, getColumnScale(field.getMetadata()));
                   epochVector.setSafe(curRowIndex, timestampWrapper.getEpoch());
                   fractionVector.setSafe(curRowIndex, timestampWrapper.getFraction());
                   timezoneVector.setSafe(
@@ -670,7 +682,8 @@ private float convertRowToArrow(
             {
               DateDayVector dateDayVector = (DateDayVector) vector;
               // Expect days past the epoch
-              int intValue = DataValidationUtil.validateAndParseDate(value);
+              int intValue =
+                  DataValidationUtil.validateAndParseDate(stats.getColumnDisplayName(), value);
               dateDayVector.setSafe(curRowIndex, intValue);
               stats.addIntValue(BigInteger.valueOf(intValue));
               rowBufferSize += 4;
@@ -682,7 +695,7 @@ private float convertRowToArrow(
                 {
                   BigInteger timeInScale =
                       DataValidationUtil.validateAndParseTime(
-                          value, getColumnScale(field.getMetadata()));
+                          stats.getColumnDisplayName(), value, getColumnScale(field.getMetadata()));
                   stats.addIntValue(timeInScale);
                   ((IntVector) vector).setSafe(curRowIndex, timeInScale.intValue());
                   stats.addIntValue(timeInScale);
@@ -693,7 +706,7 @@ private float convertRowToArrow(
                 {
                   BigInteger timeInScale =
                       DataValidationUtil.validateAndParseTime(
-                          value, getColumnScale(field.getMetadata()));
+                          stats.getColumnDisplayName(), value, getColumnScale(field.getMetadata()));
                   ((BigIntVector) vector).setSafe(curRowIndex, timeInScale.longValue());
                   stats.addIntValue(timeInScale);
                   rowBufferSize += 8;
@@ -705,7 +718,8 @@ private float convertRowToArrow(
             break;
           case BOOLEAN:
             {
-              int intValue = DataValidationUtil.validateAndParseBoolean(value);
+              int intValue =
+                  DataValidationUtil.validateAndParseBoolean(stats.getColumnDisplayName(), value);
               ((BitVector) vector).setSafe(curRowIndex, intValue);
               rowBufferSize += 0.125;
               stats.addIntValue(BigInteger.valueOf(intValue));
@@ -715,13 +729,16 @@ private float convertRowToArrow(
             String maxLengthString = field.getMetadata().get(COLUMN_BYTE_LENGTH);
             byte[] bytes =
                 DataValidationUtil.validateAndParseBinary(
-                    value, Optional.ofNullable(maxLengthString).map(Integer::parseInt));
+                    stats.getColumnDisplayName(),
+                    value,
+                    Optional.ofNullable(maxLengthString).map(Integer::parseInt));
             ((VarBinaryVector) vector).setSafe(curRowIndex, bytes);
             stats.addStrValue(new String(bytes, StandardCharsets.UTF_8));
             rowBufferSize += bytes.length;
             break;
           case REAL:
-            double doubleValue = DataValidationUtil.validateAndParseReal(value);
+            double doubleValue =
+                DataValidationUtil.validateAndParseReal(stats.getColumnDisplayName(), value);
             ((Float8Vector) vector).setSafe(curRowIndex, doubleValue);
             stats.addRealValue(doubleValue);
             rowBufferSize += 8;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 61fe83e97..b672be213 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -87,18 +87,19 @@ private static SnowflakeDateTimeFormat createDateTimeFormatter() {
    * @return JSON tree representing the input
    */
   private static JsonNode validateAndParseSemiStructuredAsJsonTree(
-      Object input, String snowflakeType) {
+      String columnName, Object input, String snowflakeType) {
     if (input instanceof String) {
       try {
         return objectMapper.readTree((String) input);
       } catch (JsonProcessingException e) {
-        throw valueFormatNotAllowedException(input, snowflakeType, "Not a valid JSON");
+        throw valueFormatNotAllowedException(columnName, input, snowflakeType, "Not a valid JSON");
       }
     } else if (isAllowedSemiStructuredType(input)) {
       return objectMapper.valueToTree(input);
     }
 
     throw typeNotAllowedException(
+        columnName,
         input.getClass(),
         snowflakeType,
         new String[] {
@@ -118,11 +119,13 @@ private static JsonNode validateAndParseSemiStructuredAsJsonTree(
    * @param input Object to validate
    * @return JSON string representing the input
    */
-  static String validateAndParseVariant(Object input) {
-    String output = validateAndParseSemiStructuredAsJsonTree(input, "VARIANT").toString();
+  static String validateAndParseVariant(String columnName, Object input) {
+    String output =
+        validateAndParseSemiStructuredAsJsonTree(columnName, input, "VARIANT").toString();
     int stringLength = output.getBytes(StandardCharsets.UTF_8).length;
     if (stringLength > MAX_SEMI_STRUCTURED_LENGTH) {
       throw valueFormatNotAllowedException(
+          columnName,
           input,
           "VARIANT",
           String.format(
@@ -242,8 +245,8 @@ static boolean isAllowedSemiStructuredType(Object o) {
    * @param input Object to validate
    * @return JSON array representing the input
    */
-  static String validateAndParseArray(Object input) {
-    JsonNode jsonNode = validateAndParseSemiStructuredAsJsonTree(input, "ARRAY");
+  static String validateAndParseArray(String columnName, Object input) {
+    JsonNode jsonNode = validateAndParseSemiStructuredAsJsonTree(columnName, input, "ARRAY");
 
     // Non-array values are ingested as single-element arrays, mimicking the Worksheets behavior
     if (!jsonNode.isArray()) {
@@ -255,6 +258,7 @@ static String validateAndParseArray(Object input) {
     int stringLength = output.getBytes(StandardCharsets.UTF_8).length;
     if (stringLength > MAX_SEMI_STRUCTURED_LENGTH) {
       throw valueFormatNotAllowedException(
+          columnName,
           jsonNode.toString(),
           "ARRAY",
           String.format(
@@ -271,10 +275,10 @@ static String validateAndParseArray(Object input) {
    * @param input Object to validate
    * @return JSON object representing the input
    */
-  static String validateAndParseObject(Object input) {
-    JsonNode jsonNode = validateAndParseSemiStructuredAsJsonTree(input, "OBJECT");
+  static String validateAndParseObject(String columnName, Object input) {
+    JsonNode jsonNode = validateAndParseSemiStructuredAsJsonTree(columnName, input, "OBJECT");
     if (!jsonNode.isObject()) {
-      throw valueFormatNotAllowedException(jsonNode, "OBJECT", "Not an object");
+      throw valueFormatNotAllowedException(columnName, jsonNode, "OBJECT", "Not an object");
     }
 
     String output = jsonNode.toString();
@@ -282,6 +286,7 @@ static String validateAndParseObject(Object input) {
     int stringLength = output.getBytes(StandardCharsets.UTF_8).length;
     if (stringLength > MAX_SEMI_STRUCTURED_LENGTH) {
       throw valueFormatNotAllowedException(
+          columnName,
           output,
           "OBJECT",
           String.format(
@@ -311,7 +316,7 @@ static String validateAndParseObject(Object input) {
    *     scale
    */
   static TimestampWrapper validateAndParseTimestampNtzSb16(
-      Object input, int scale, boolean ignoreTimezone) {
+      String columnName, Object input, int scale, boolean ignoreTimezone) {
     String valueString;
     if (input instanceof String) valueString = (String) input;
     else if (input instanceof LocalDate) valueString = input.toString();
@@ -328,6 +333,7 @@ else if (input instanceof OffsetDateTime)
               : DateTimeFormatter.ISO_DATE_TIME.format((OffsetDateTime) input);
     else
       throw typeNotAllowedException(
+          columnName,
           input.getClass(),
           "TIMESTAMP",
           new String[] {"String", "LocalDate", "LocalDateTime", "ZonedDateTime", "OffsetDateTime"});
@@ -348,6 +354,7 @@ else if (input instanceof OffsetDateTime)
           epoch, fractionInScale * Power10.intTable[9 - scale], timeInScale);
     } else {
       throw valueFormatNotAllowedException(
+          columnName,
           input.toString(),
           "TIMESTAMP",
           "Not a valid timestamp, see"
@@ -385,7 +392,7 @@ private static int getFractionFromTimestamp(SFTimestamp input) {
    * @return TimestampWrapper with epoch seconds, fractional seconds, and epoch time in the column
    *     scale
    */
-  static TimestampWrapper validateAndParseTimestampTz(Object input, int scale) {
+  static TimestampWrapper validateAndParseTimestampTz(String columnName, Object input, int scale) {
 
     String stringInput;
     if (input instanceof String) stringInput = (String) input;
@@ -398,6 +405,7 @@ else if (input instanceof OffsetDateTime)
       stringInput = DateTimeFormatter.ISO_DATE_TIME.format((OffsetDateTime) input);
     else
       throw typeNotAllowedException(
+          columnName,
           input.getClass(),
           "TIMESTAMP",
           new String[] {"String", "LocalDate", "LocalDateTime", "ZonedDateTime", "OffsetDateTime"});
@@ -417,6 +425,7 @@ else if (input instanceof OffsetDateTime)
           timestamp);
     } else {
       throw valueFormatNotAllowedException(
+          columnName,
           input.toString(),
           "TIMESTAMP",
           "Not a valid timestamp, see"
@@ -441,7 +450,8 @@ else if (input instanceof OffsetDateTime)
    *     maximum allowed by Snowflake
    *     (https://docs.snowflake.com/en/sql-reference/data-types-text.html#varchar)
    */
-  static String validateAndParseString(Object input, Optional<Integer> maxLengthOptional) {
+  static String validateAndParseString(
+      String columnName, Object input, Optional<Integer> maxLengthOptional) {
     String output;
     if (input instanceof String) {
       output = (String) input;
@@ -451,12 +461,16 @@ static String validateAndParseString(Object input, Optional<Integer> maxLengthOp
       output = input.toString();
     } else {
       throw typeNotAllowedException(
-          input.getClass(), "STRING", new String[] {"String", "Number", "boolean", "char"});
+          columnName,
+          input.getClass(),
+          "STRING",
+          new String[] {"String", "Number", "boolean", "char"});
     }
     int maxLength = maxLengthOptional.orElse(BYTES_16_MB);
 
     if (output.length() > maxLength) {
       throw valueFormatNotAllowedException(
+          columnName,
           input,
           "STRING",
           String.format("String too long: length=%d maxLength=%d", output.length(), maxLength));
@@ -472,7 +486,7 @@ static String validateAndParseString(Object input, Optional<Integer> maxLengthOp
    * <li>BigInteger, BigDecimal
    * <li>String
    */
-  static BigDecimal validateAndParseBigDecimal(Object input) {
+  static BigDecimal validateAndParseBigDecimal(String columnName, Object input) {
     if (input instanceof BigDecimal) {
       return (BigDecimal) input;
     } else if (input instanceof BigInteger) {
@@ -488,10 +502,11 @@ static BigDecimal validateAndParseBigDecimal(Object input) {
       try {
         return new BigDecimal((String) input);
       } catch (NumberFormatException e) {
-        throw valueFormatNotAllowedException(input, "NUMBER", "Not a valid number");
+        throw valueFormatNotAllowedException(columnName, input, "NUMBER", "Not a valid number");
       }
     } else {
       throw typeNotAllowedException(
+          columnName,
           input.getClass(),
           "NUMBER",
           new String[] {
@@ -511,7 +526,7 @@ static BigDecimal validateAndParseBigDecimal(Object input) {
    *   <li>ZonedDateTime
    * </ul>
    */
-  static int validateAndParseDate(Object input) {
+  static int validateAndParseDate(String columnName, Object input) {
     String inputString;
     if (input instanceof String) {
       inputString = (String) input;
@@ -525,6 +540,7 @@ static int validateAndParseDate(Object input) {
       inputString = ((OffsetDateTime) input).toLocalDate().toString();
     } else {
       throw typeNotAllowedException(
+          columnName,
           input.getClass(),
           "DATE",
           new String[] {"String", "LocalDate", "LocalDateTime", "ZonedDateTime", "OffsetDateTime"});
@@ -534,6 +550,7 @@ static int validateAndParseDate(Object input) {
         createDateTimeFormatter().parse(inputString, GMT, 0, DATE | TIMESTAMP, true, null);
     if (timestamp == null)
       throw valueFormatNotAllowedException(
+          columnName,
           input,
           "DATE",
           "Not a valid date, see"
@@ -556,7 +573,8 @@ static int validateAndParseDate(Object input) {
    *     BINARY column
    * @return Validated array
    */
-  static byte[] validateAndParseBinary(Object input, Optional<Integer> maxLengthOptional) {
+  static byte[] validateAndParseBinary(
+      String columnName, Object input, Optional<Integer> maxLengthOptional) {
     byte[] output;
     if (input instanceof byte[]) {
       output = (byte[]) input;
@@ -564,15 +582,17 @@ static byte[] validateAndParseBinary(Object input, Optional<Integer> maxLengthOp
       try {
         output = DatatypeConverter.parseHexBinary((String) input);
       } catch (IllegalArgumentException e) {
-        throw valueFormatNotAllowedException(input, "BINARY", "Not a valid hex string");
+        throw valueFormatNotAllowedException(columnName, input, "BINARY", "Not a valid hex string");
       }
     } else {
-      throw typeNotAllowedException(input.getClass(), "BINARY", new String[] {"byte[]", "String"});
+      throw typeNotAllowedException(
+          columnName, input.getClass(), "BINARY", new String[] {"byte[]", "String"});
     }
 
     int maxLength = maxLengthOptional.orElse(BYTES_8_MB);
     if (output.length > maxLength) {
       throw valueFormatNotAllowedException(
+          columnName,
           String.format("byte[%d]", output.length),
           "BINARY",
           String.format("Binary too long: length=%d maxLength=%d", output.length, maxLength));
@@ -590,7 +610,7 @@ static byte[] validateAndParseBinary(Object input, Optional<Integer> maxLengthOp
    *   <li>OffsetTime
    * </ul>
    */
-  static BigInteger validateAndParseTime(Object input, int scale) {
+  static BigInteger validateAndParseTime(String columnName, Object input, int scale) {
     String stringInput;
     if (input instanceof String) {
       stringInput = (String) input;
@@ -600,7 +620,7 @@ static BigInteger validateAndParseTime(Object input, int scale) {
       stringInput = ((OffsetTime) input).toLocalTime().toString();
     } else {
       throw typeNotAllowedException(
-          input.getClass(), "TIME", new String[] {"String", "LocalTime", "OffsetTime"});
+          columnName, input.getClass(), "TIME", new String[] {"String", "LocalTime", "OffsetTime"});
     }
 
     SFTimestamp timestamp =
@@ -608,6 +628,7 @@ static BigInteger validateAndParseTime(Object input, int scale) {
             .parse(stringInput, GMT, 0, SnowflakeDateTimeFormat.TIME, true, null);
     if (timestamp == null) {
       throw valueFormatNotAllowedException(
+          columnName,
           input,
           "TIME",
           "Not a valid time, see"
@@ -632,7 +653,7 @@ static BigInteger validateAndParseTime(Object input, int scale) {
    *
    * @param input
    */
-  static double validateAndParseReal(Object input) {
+  static double validateAndParseReal(String columnName, Object input) {
     if (input instanceof Float) {
       return Double.parseDouble(input.toString());
     } else if (input instanceof Number) {
@@ -651,12 +672,13 @@ static double validateAndParseReal(Object input) {
           case "-inf":
             return Double.NEGATIVE_INFINITY;
           default:
-            throw valueFormatNotAllowedException(input, "REAL", "Not a valid decimal number");
+            throw valueFormatNotAllowedException(
+                    columnName, input, "REAL", "Not a valid decimal number");
         }
       }
     }
-
-    throw typeNotAllowedException(input.getClass(), "REAL", new String[] {"Number", "String"});
+    throw typeNotAllowedException(
+        columnName, input.getClass(), "REAL", new String[] {"Number", "String"});
   }
 
   /**
@@ -673,17 +695,17 @@ static double validateAndParseReal(Object input) {
    * @param input Input to be converted
    * @return 1 or 0 where 1=true, 0=false
    */
-  static int validateAndParseBoolean(Object input) {
+  static int validateAndParseBoolean(String columnName, Object input) {
     if (input instanceof Boolean) {
       return (boolean) input ? 1 : 0;
     } else if (input instanceof Number) {
       return new BigDecimal(input.toString()).compareTo(BigDecimal.ZERO) == 0 ? 0 : 1;
     } else if (input instanceof String) {
-      return convertStringToBoolean((String) input) ? 1 : 0;
+      return convertStringToBoolean(columnName, (String) input) ? 1 : 0;
     }
 
     throw typeNotAllowedException(
-        input.getClass(), "BOOLEAN", new String[] {"boolean", "Number", "String"});
+        columnName, input.getClass(), "BOOLEAN", new String[] {"boolean", "Number", "String"});
   }
 
   static void checkValueInRange(BigDecimal bigDecimalValue, int scale, int precision) {
@@ -700,10 +722,11 @@ static void checkValueInRange(BigDecimal bigDecimalValue, int scale, int precisi
   static Set<String> allowedBooleanStringsLowerCased =
       Sets.newHashSet("1", "0", "yes", "no", "y", "n", "t", "f", "true", "false", "on", "off");
 
-  private static boolean convertStringToBoolean(String value) {
+  private static boolean convertStringToBoolean(String columnName, String value) {
     String lowerCasedValue = value.toLowerCase();
     if (!allowedBooleanStringsLowerCased.contains(lowerCasedValue)) {
       throw valueFormatNotAllowedException(
+          columnName,
           value,
           "BOOLEAN",
           "Not a valid boolean, see"
@@ -726,12 +749,12 @@ private static boolean convertStringToBoolean(String value) {
    * @param allowedJavaTypes Java types supported for the Java type
    */
   private static SFException typeNotAllowedException(
-      Class<?> javaType, String snowflakeType, String[] allowedJavaTypes) {
+      String columnName, Class<?> javaType, String snowflakeType, String[] allowedJavaTypes) {
     return new SFException(
         ErrorCode.INVALID_ROW,
         String.format(
-            "Object of type %s cannot be ingested into Snowflake column of type %s",
-            javaType.getName(), snowflakeType),
+            "Object of type %s cannot be ingested into Snowflake column %s of type %s",
+            javaType.getName(), columnName, snowflakeType),
         String.format(
             String.format("Allowed Java types: %s", String.join(", ", allowedJavaTypes))));
   }
@@ -744,12 +767,13 @@ private static SFException typeNotAllowedException(
    * @param snowflakeType Snowflake column type
    */
   private static SFException valueFormatNotAllowedException(
-      Object value, String snowflakeType, String reason) {
+      String columnName, Object value, String snowflakeType, String reason) {
     return new SFException(
         ErrorCode.INVALID_ROW,
         sanitizeValueForExceptionMessage(value),
         String.format(
-            "Value cannot be ingested into Snowflake column %s: %s", snowflakeType, reason));
+            "Value cannot be ingested into Snowflake column %s of type %s: %s",
+            columnName, snowflakeType, reason));
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
index ebd958b75..cc6130531 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
@@ -59,7 +59,8 @@ static ParquetBufferValue parseColumnValueToParquet(
           AbstractRowBuffer.ColumnPhysicalType.valueOf(columnMetadata.getPhysicalType());
       switch (typeName) {
         case BOOLEAN:
-          int intValue = DataValidationUtil.validateAndParseBoolean(value);
+          int intValue =
+              DataValidationUtil.validateAndParseBoolean(columnMetadata.getName(), value);
           value = intValue > 0;
           stats.addIntValue(BigInteger.valueOf(intValue));
           size = 1;
@@ -67,6 +68,7 @@ static ParquetBufferValue parseColumnValueToParquet(
         case INT32:
           int intVal =
               getInt32Value(
+                  columnMetadata.getName(),
                   value,
                   columnMetadata.getScale(),
                   Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
@@ -79,6 +81,7 @@ static ParquetBufferValue parseColumnValueToParquet(
         case INT64:
           long longValue =
               getInt64Value(
+                  columnMetadata.getName(),
                   value,
                   columnMetadata.getScale(),
                   Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
@@ -89,7 +92,8 @@ static ParquetBufferValue parseColumnValueToParquet(
           size = 8;
           break;
         case DOUBLE:
-          double doubleValue = DataValidationUtil.validateAndParseReal(value);
+          double doubleValue =
+              DataValidationUtil.validateAndParseReal(columnMetadata.getName(), value);
           value = doubleValue;
           stats.addRealValue(doubleValue);
           size = 8;
@@ -107,6 +111,7 @@ static ParquetBufferValue parseColumnValueToParquet(
         case FIXED_LEN_BYTE_ARRAY:
           BigInteger intRep =
               getSb16Value(
+                  columnMetadata.getName(),
                   value,
                   columnMetadata.getScale(),
                   Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
@@ -140,6 +145,7 @@ static ParquetBufferValue parseColumnValueToParquet(
    * @return parsed int32 value
    */
   private static int getInt32Value(
+      String columnName,
       Object value,
       @Nullable Integer scale,
       Integer precision,
@@ -148,14 +154,15 @@ private static int getInt32Value(
     int intVal;
     switch (logicalType) {
       case DATE:
-        intVal = DataValidationUtil.validateAndParseDate(value);
+        intVal = DataValidationUtil.validateAndParseDate(columnName, value);
         break;
       case TIME:
         Utils.assertNotNull("Unexpected null scale for TIME data type", scale);
-        intVal = DataValidationUtil.validateAndParseTime(value, scale).intValue();
+        intVal = DataValidationUtil.validateAndParseTime(columnName, value, scale).intValue();
         break;
       case FIXED:
-        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
+        BigDecimal bigDecimalValue =
+            DataValidationUtil.validateAndParseBigDecimal(columnName, value);
         bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
         DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
         intVal = bigDecimalValue.intValue();
@@ -177,6 +184,7 @@ private static int getInt32Value(
    * @return parsed int64 value
    */
   private static long getInt64Value(
+      String columnName,
       Object value,
       int scale,
       int precision,
@@ -186,19 +194,20 @@ private static long getInt64Value(
     switch (logicalType) {
       case TIME:
         Utils.assertNotNull("Unexpected null scale for TIME data type", scale);
-        longValue = DataValidationUtil.validateAndParseTime(value, scale).longValue();
+        longValue = DataValidationUtil.validateAndParseTime(columnName, value, scale).longValue();
         break;
       case TIMESTAMP_LTZ:
       case TIMESTAMP_NTZ:
         boolean ignoreTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
         longValue =
-            DataValidationUtil.validateAndParseTimestampNtzSb16(value, scale, ignoreTimezone)
+            DataValidationUtil.validateAndParseTimestampNtzSb16(
+                    columnName, value, scale, ignoreTimezone)
                 .getTimeInScale()
                 .longValue();
         break;
       case TIMESTAMP_TZ:
         longValue =
-            DataValidationUtil.validateAndParseTimestampTz(value, scale)
+            DataValidationUtil.validateAndParseTimestampTz(columnName, value, scale)
                 .getSfTimestamp()
                 .orElseThrow(
                     () ->
@@ -210,7 +219,8 @@ private static long getInt64Value(
                 .longValue();
         break;
       case FIXED:
-        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
+        BigDecimal bigDecimalValue =
+            DataValidationUtil.validateAndParseBigDecimal(columnName, value);
         bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
         DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
         longValue = bigDecimalValue.longValue();
@@ -232,6 +242,7 @@ private static long getInt64Value(
    * @return parsed int64 value
    */
   private static BigInteger getSb16Value(
+      String columnName,
       Object value,
       int scale,
       int precision,
@@ -239,7 +250,7 @@ private static BigInteger getSb16Value(
       AbstractRowBuffer.ColumnPhysicalType physicalType) {
     switch (logicalType) {
       case TIMESTAMP_TZ:
-        return DataValidationUtil.validateAndParseTimestampTz(value, scale)
+        return DataValidationUtil.validateAndParseTimestampTz(columnName, value, scale)
             .getSfTimestamp()
             .orElseThrow(
                 () ->
@@ -251,10 +262,12 @@ private static BigInteger getSb16Value(
       case TIMESTAMP_LTZ:
       case TIMESTAMP_NTZ:
         boolean ignoreTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
-        return DataValidationUtil.validateAndParseTimestampNtzSb16(value, scale, ignoreTimezone)
+        return DataValidationUtil.validateAndParseTimestampNtzSb16(
+                columnName, value, scale, ignoreTimezone)
             .getTimeInScale();
       case FIXED:
-        BigDecimal bigDecimalValue = DataValidationUtil.validateAndParseBigDecimal(value);
+        BigDecimal bigDecimalValue =
+            DataValidationUtil.validateAndParseBigDecimal(columnName, value);
         // explicitly match the BigDecimal input scale with the Snowflake data type scale
         bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
         DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
@@ -297,13 +310,13 @@ private static String getBinaryValue(
     if (logicalType.isObject()) {
       switch (logicalType) {
         case OBJECT:
-          str = DataValidationUtil.validateAndParseObject(value);
+          str = DataValidationUtil.validateAndParseObject(columnMetadata.getName(), value);
           break;
         case VARIANT:
-          str = DataValidationUtil.validateAndParseVariant(value);
+          str = DataValidationUtil.validateAndParseVariant(columnMetadata.getName(), value);
           break;
         case ARRAY:
-          str = DataValidationUtil.validateAndParseArray(value);
+          str = DataValidationUtil.validateAndParseArray(columnMetadata.getName(), value);
           break;
         default:
           throw new SFException(
@@ -313,7 +326,7 @@ private static String getBinaryValue(
       String maxLengthString = columnMetadata.getLength().toString();
       str =
           DataValidationUtil.validateAndParseString(
-              value, Optional.of(maxLengthString).map(Integer::parseInt));
+              columnMetadata.getName(), value, Optional.of(maxLengthString).map(Integer::parseInt));
       stats.addStrValue(str);
     }
     return str;
@@ -331,7 +344,7 @@ private static byte[] getBinaryValueForLogicalBinary(
     String maxLengthString = columnMetadata.getLength().toString();
     byte[] bytes =
         DataValidationUtil.validateAndParseBinary(
-            value, Optional.of(maxLengthString).map(Integer::parseInt));
+                columnMetadata.getName(), value, Optional.of(maxLengthString).map(Integer::parseInt));
 
     String str = new String(bytes, StandardCharsets.UTF_8);
     stats.addStrValue(str);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 7b95c69aa..1d6827349 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -18,6 +18,8 @@
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -36,7 +38,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.function.Function;
 import javax.xml.bind.DatatypeConverter;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
@@ -46,10 +47,6 @@
 public class DataValidationUtilTest {
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
-  private void expectError(ErrorCode expectedErrorCode, Function func, Object args) {
-    expectError(expectedErrorCode, () -> func.apply(args));
-  }
-
   private void expectErrorCodeAndMessage(
       ErrorCode expectedErrorCode, String expectedExceptionMessage, Runnable action) {
     try {
@@ -70,164 +67,198 @@ private void expectError(ErrorCode expectedErrorCode, Runnable action) {
 
   @Test
   public void testValidateAndParseTime() {
-    assertEquals(5L, validateAndParseTime("00:00:05", 0).longValueExact());
-    assertEquals(5000L, validateAndParseTime("00:00:05", 3).longValueExact());
-    assertEquals(5000L, validateAndParseTime("00:00:05.000", 3).longValueExact());
-    assertEquals(5123L, validateAndParseTime("00:00:05.123", 3).longValueExact());
-    assertEquals(5123L, validateAndParseTime("00:00:05.123456", 3).longValueExact());
-    assertEquals(5123456789L, validateAndParseTime("00:00:05.123456789", 9).longValueExact());
+    assertEquals(5L, validateAndParseTime("COL", "00:00:05", 0).longValueExact());
+    assertEquals(5000L, validateAndParseTime("COL", "00:00:05", 3).longValueExact());
+    assertEquals(5000L, validateAndParseTime("COL", "00:00:05.000", 3).longValueExact());
+    assertEquals(5123L, validateAndParseTime("COL", "00:00:05.123", 3).longValueExact());
+    assertEquals(5123L, validateAndParseTime("COL", "00:00:05.123456", 3).longValueExact());
+    assertEquals(
+        5123456789L, validateAndParseTime("COL", "00:00:05.123456789", 9).longValueExact());
 
-    assertEquals(72L, validateAndParseTime("72", 0).longValueExact());
-    assertEquals(72000L, validateAndParseTime("72", 3).longValueExact());
+    assertEquals(72L, validateAndParseTime("COL", "72", 0).longValueExact());
+    assertEquals(72000L, validateAndParseTime("COL", "72", 3).longValueExact());
 
     // Timestamps are rejected
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2/18/2008 02:36:48", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57:01", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57:01 +07:00", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57:01 +0700", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57:01-07", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57:01-07:00", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57:01.123456", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2/18/2008 02:36:48", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20:57", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20:57:01", 9));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20:57:01 +07:00", 9));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20:57:01 +0700", 9));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20:57:01-07", 9));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20:57:01-07:00", 9));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20:57:01.123456", 9));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("2013-04-28 20:57:01.123456789 +07:00", 9));
+        () -> validateAndParseTime("COL", "2013-04-28 20:57:01.123456789 +07:00", 9));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("2013-04-28 20:57:01.123456789 +0700", 9));
+        () -> validateAndParseTime("COL", "2013-04-28 20:57:01.123456789 +0700", 9));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57:01.123456789+07", 9));
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime("COL", "2013-04-28 20:57:01.123456789+07", 9));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("2013-04-28 20:57:01.123456789+07:00", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28 20:57+07:00", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28T20", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28T20:57", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28T20:57:01", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28T20:57:01-07:00", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28T20:57:01.123456", 9));
+        () -> validateAndParseTime("COL", "2013-04-28 20:57:01.123456789+07:00", 9));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20:57+07:00", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28T20", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28T20:57", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28T20:57:01", 9));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28T20:57:01-07:00", 9));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28T20:57:01.123456", 9));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("2013-04-28T20:57:01.123456789+07:00", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28T20:57+07:00", 9));
+        () -> validateAndParseTime("COL", "2013-04-28T20:57:01.123456789+07:00", 9));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime("Mon Jul 08 18:09:51 +0000 2013", 9));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28T20:57+07:00", 9));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime("Thu, 21 Dec 2000 04:01:07 PM", 9));
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime("COL", "Mon Jul 08 18:09:51 +0000 2013", 9));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime("Thu, 21 Dec 2000 04:01:07 PM +0200", 9));
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime("COL", "Thu, 21 Dec 2000 04:01:07 PM", 9));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("Thu, 21 Dec 2000 04:01:07.123456789 PM", 9));
+        () -> validateAndParseTime("COL", "Thu, 21 Dec 2000 04:01:07 PM +0200", 9));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("Thu, 21 Dec 2000 04:01:07.123456789 PM +0200", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("Thu, 21 Dec 2000 16:01:07", 9));
+        () -> validateAndParseTime("COL", "Thu, 21 Dec 2000 04:01:07.123456789 PM", 9));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime("Thu, 21 Dec 2000 16:01:07 +0200", 9));
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime("COL", "Thu, 21 Dec 2000 04:01:07.123456789 PM +0200", 9));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "Thu, 21 Dec 2000 16:01:07", 9));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime("COL", "Thu, 21 Dec 2000 16:01:07 +0200", 9));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("Thu, 21 Dec 2000 16:01:07.123456789", 9));
+        () -> validateAndParseTime("COL", "Thu, 21 Dec 2000 16:01:07.123456789", 9));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("Thu, 21 Dec 2000 16:01:07.123456789 +0200", 9));
+        () -> validateAndParseTime("COL", "Thu, 21 Dec 2000 16:01:07.123456789 +0200", 9));
 
     // Dates are rejected
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("2013-04-28", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("17-DEC-1980", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("12/17/1980", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "17-DEC-1980", 9));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "12/17/1980", 9));
 
     // Test forbidden values
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(LocalDate.now(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(LocalDateTime.now(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(OffsetDateTime.now(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(ZonedDateTime.now(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(new Date(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(1.5f, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(1.5, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("1.5", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("1.0", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(new Object(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(false, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("foo", 3));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime(java.sql.Time.valueOf("20:57:00"), 3));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime(java.sql.Date.valueOf("2010-11-03"), 3));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime(java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(BigInteger.ZERO, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime(BigDecimal.ZERO, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime('c', 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", LocalDate.now(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", LocalDateTime.now(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", OffsetDateTime.now(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", ZonedDateTime.now(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", new Date(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", 1.5f, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", 1.5, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "1.5", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "1.0", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", new Object(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", false, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "foo", 3));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime("COL", java.sql.Time.valueOf("20:57:00"), 3));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime("COL", java.sql.Date.valueOf("2010-11-03"), 3));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTime("COL", java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", BigInteger.ZERO, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", BigDecimal.ZERO, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", 'c', 3));
   }
 
   @Test
   public void testValidateAndParseTimestampNtzSb16() {
     assertEquals(
         new TimestampWrapper(1609462800, 123000000, new BigInteger("1609462800123000000")),
-        DataValidationUtil.validateAndParseTimestampNtzSb16("2021-01-01 01:00:00.123", 9, true));
+        DataValidationUtil.validateAndParseTimestampNtzSb16(
+            "COL", "2021-01-01 01:00:00.123", 9, true));
 
     // Time formats are not supported
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampNtzSb16("20:57:01.123456789+07:00", 3, false));
+        () -> validateAndParseTimestampNtzSb16("COL", "20:57:01.123456789+07:00", 3, false));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampNtzSb16("20:57:01.123456789", 3, false));
+        () -> validateAndParseTimestampNtzSb16("COL", "20:57:01.123456789", 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("20:57:01", 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("20:57", 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", "20:57:01", 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", "20:57", 3, false));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampNtzSb16("07:57:01.123456789 AM", 3, false));
+        () -> validateAndParseTimestampNtzSb16("COL", "07:57:01.123456789 AM", 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("04:01:07 AM", 3, false));
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestampNtzSb16("COL", "04:01:07 AM", 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("04:01 AM", 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", "04:01 AM", 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("04:01 PM", 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", "04:01 PM", 3, false));
 
     // Test forbidden values
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(LocalTime.now(), 3, false));
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestampNtzSb16("COL", LocalTime.now(), 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestampNtzSb16("COL", OffsetTime.now(), 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(OffsetTime.now(), 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", new Date(), 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(new Date(), 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(1.5f, 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(1.5, 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("1.5", 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("1.0", 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", 1.5f, 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(new Object(), 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(false, 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("", 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("foo", 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", 1.5, 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", "1.5", 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", "1.0", 3, false));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampNtzSb16(java.sql.Time.valueOf("20:57:00"), 3, false));
+        () -> validateAndParseTimestampNtzSb16("COL", new Object(), 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", false, 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", "", 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", "foo", 3, false));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampNtzSb16(java.sql.Date.valueOf("2010-11-03"), 3, false));
+        () -> validateAndParseTimestampNtzSb16("COL", java.sql.Time.valueOf("20:57:00"), 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () ->
+            validateAndParseTimestampNtzSb16("COL", java.sql.Date.valueOf("2010-11-03"), 3, false));
     expectError(
         ErrorCode.INVALID_ROW,
         () ->
             validateAndParseTimestampNtzSb16(
-                java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3, false));
+                "COL", java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestampNtzSb16("COL", BigInteger.ZERO, 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(BigInteger.ZERO, 3, false));
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestampNtzSb16("COL", BigDecimal.ZERO, 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16(BigDecimal.ZERO, 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16('c', 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", 'c', 3, false));
   }
 
   @Test
   public void testValidateAndPareTimestampTz() {
     TimestampWrapper result =
-        DataValidationUtil.validateAndParseTimestampTz("2021-01-01 01:00:00.123 +0100", 4);
+        DataValidationUtil.validateAndParseTimestampTz("COL", "2021-01-01 01:00:00.123 +0100", 4);
     assertEquals(1609459200, result.getEpoch());
     assertEquals(123000000, result.getFraction());
     assertEquals(Optional.of(3600000), result.getTimezoneOffset());
@@ -235,92 +266,106 @@ public void testValidateAndPareTimestampTz() {
 
     // Time formats are not supported
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("20:57:01.123456789+07:00", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("20:57:01.123456789", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("20:57:01", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("20:57", 3));
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestampTz("COL", "20:57:01.123456789+07:00", 3));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "20:57:01.123456789", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "20:57:01", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "20:57", 3));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("07:57:01.123456789 AM", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("04:01:07 AM", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("04:01 AM", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("04:01 PM", 3));
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestampTz("COL", "07:57:01.123456789 AM", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "04:01:07 AM", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "04:01 AM", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "04:01 PM", 3));
 
     // Test forbidden values
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(LocalTime.now(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(OffsetTime.now(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(new Date(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(1.5f, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(1.5, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("1.5", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("1.0", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(new Object(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(false, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("foo", 3));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", LocalTime.now(), 3));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", OffsetTime.now(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", new Date(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", 1.5f, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", 1.5, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "1.5", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "1.0", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", new Object(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", false, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "", 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "foo", 3));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampTz(java.sql.Time.valueOf("20:57:00"), 3));
+        () -> validateAndParseTimestampTz("COL", java.sql.Time.valueOf("20:57:00"), 3));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampTz(java.sql.Date.valueOf("2010-11-03"), 3));
+        () -> validateAndParseTimestampTz("COL", java.sql.Date.valueOf("2010-11-03"), 3));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampTz(java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(BigInteger.ZERO, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz(BigDecimal.ZERO, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz('c', 3));
+        () ->
+            validateAndParseTimestampTz(
+                "COL", java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", BigInteger.ZERO, 3));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", BigDecimal.ZERO, 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", 'c', 3));
   }
 
   @Test
   public void testValidateAndParseBigDecimal() {
-    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal("1"));
+    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal("COL", "1"));
     assertEquals(
-        new BigDecimal("1000").toBigInteger(), validateAndParseBigDecimal("1e3").toBigInteger());
+        new BigDecimal("1000").toBigInteger(),
+        validateAndParseBigDecimal("COL", "1e3").toBigInteger());
     assertEquals(
-        new BigDecimal("-1000").toBigInteger(), validateAndParseBigDecimal("-1e3").toBigInteger());
+        new BigDecimal("-1000").toBigInteger(),
+        validateAndParseBigDecimal("COL", "-1e3").toBigInteger());
     assertEquals(
-        new BigDecimal("1").toBigInteger(), validateAndParseBigDecimal("1e0").toBigInteger());
+        new BigDecimal("1").toBigInteger(),
+        validateAndParseBigDecimal("COL", "1e0").toBigInteger());
     assertEquals(
-        new BigDecimal("-1").toBigInteger(), validateAndParseBigDecimal("-1e0").toBigInteger());
+        new BigDecimal("-1").toBigInteger(),
+        validateAndParseBigDecimal("COL", "-1e0").toBigInteger());
     assertEquals(
-        new BigDecimal("123").toBigInteger(), validateAndParseBigDecimal("1.23e2").toBigInteger());
+        new BigDecimal("123").toBigInteger(),
+        validateAndParseBigDecimal("COL", "1.23e2").toBigInteger());
     assertEquals(
         new BigDecimal("123.4").toBigInteger(),
-        validateAndParseBigDecimal("1.234e2").toBigInteger());
+        validateAndParseBigDecimal("COL", "1.234e2").toBigInteger());
     assertEquals(
         new BigDecimal("0.1234").toBigInteger(),
-        validateAndParseBigDecimal("1.234e-1").toBigInteger());
+        validateAndParseBigDecimal("COL", "1.234e-1").toBigInteger());
     assertEquals(
         new BigDecimal("0.1234").toBigInteger(),
-        validateAndParseBigDecimal("1234e-5").toBigInteger());
+        validateAndParseBigDecimal("COL", "1234e-5").toBigInteger());
     assertEquals(
         new BigDecimal("0.1234").toBigInteger(),
-        validateAndParseBigDecimal("1234E-5").toBigInteger());
-    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal(1));
-    assertEquals(new BigDecimal("1.0"), validateAndParseBigDecimal(1D));
-    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal(1L));
-    assertEquals(new BigDecimal("1.0"), validateAndParseBigDecimal(1F));
+        validateAndParseBigDecimal("COL", "1234E-5").toBigInteger());
+    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal("COL", 1));
+    assertEquals(new BigDecimal("1.0"), validateAndParseBigDecimal("COL", 1D));
+    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal("COL", 1L));
+    assertEquals(new BigDecimal("1.0"), validateAndParseBigDecimal("COL", 1F));
     assertEquals(
-        BigDecimal.valueOf(10).pow(37), validateAndParseBigDecimal(BigDecimal.valueOf(10).pow(37)));
+        BigDecimal.valueOf(10).pow(37),
+        validateAndParseBigDecimal("COL", BigDecimal.valueOf(10).pow(37)));
     assertEquals(
         BigDecimal.valueOf(-1).multiply(BigDecimal.valueOf(10).pow(37)),
         validateAndParseBigDecimal(
-            BigInteger.valueOf(-1).multiply(BigInteger.valueOf(10).pow(37))));
+            "COL", BigInteger.valueOf(-1).multiply(BigInteger.valueOf(10).pow(37))));
 
     // Test forbidden values
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigDecimal, "honk");
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigDecimal, "0x22");
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigDecimal, true);
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigDecimal, false);
-    expectError(
-        ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigDecimal, new Object());
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigDecimal, 'a');
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBigDecimal, new byte[4]);
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBigDecimal("COL", "honk"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBigDecimal("COL", "0x22"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBigDecimal("COL", true));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBigDecimal("COL", false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBigDecimal("COL", new Object()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBigDecimal("COL", 'a'));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBigDecimal("COL", new byte[4]));
   }
 
   @Test
   public void testValidateAndParseString() {
-    assertEquals("honk", validateAndParseString("honk", Optional.empty()));
+    assertEquals("honk", validateAndParseString("COL", "honk", Optional.empty()));
 
     // Check max String length
     StringBuilder longBuilder = new StringBuilder();
@@ -328,138 +373,132 @@ public void testValidateAndParseString() {
       longBuilder.append("č"); // max string length is measured in chars, not bytes
     }
     String maxString = longBuilder.toString();
-    Assert.assertEquals(maxString, validateAndParseString(maxString, Optional.empty()));
+    Assert.assertEquals(maxString, validateAndParseString("COL", maxString, Optional.empty()));
 
     // max length - 1 should also succeed
     longBuilder.setLength(BYTES_16_MB - 1);
     String maxStringMinusOne = longBuilder.toString();
     Assert.assertEquals(
-        maxStringMinusOne, validateAndParseString(maxStringMinusOne, Optional.empty()));
+        maxStringMinusOne, validateAndParseString("COL", maxStringMinusOne, Optional.empty()));
 
     // max length + 1 should fail
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseString(longBuilder.append("aa").toString(), Optional.empty()));
+        () -> validateAndParseString("COL", longBuilder.append("aa").toString(), Optional.empty()));
 
     // Test max length validation
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString("12345", Optional.of(4)));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString(false, Optional.of(4)));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString(12345, Optional.of(4)));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString(1.2345, Optional.of(4)));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", "12345", Optional.of(4)));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", false, Optional.of(4)));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", 12345, Optional.of(4)));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", 1.2345, Optional.of(4)));
 
     // Test unsupported values
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseString(new Object(), Optional.empty()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString(new byte[] {}, Optional.of(4)));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString(new char[] {}, Optional.of(4)));
+        ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", new Object(), Optional.empty()));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", new byte[] {}, Optional.of(4)));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", new char[] {}, Optional.of(4)));
   }
 
   @Test
   public void testValidateAndParseVariant() throws Exception {
-    assertEquals("1", validateAndParseVariant(1));
-    assertEquals("1", validateAndParseVariant("1"));
-    assertEquals("1", validateAndParseVariant("                          1   "));
+    assertEquals("1", validateAndParseVariant("COL", 1));
+    assertEquals("1", validateAndParseVariant("COL", "1"));
+    assertEquals("1", validateAndParseVariant("COL", "                          1   "));
     String stringVariant = "{\"key\":1}";
-    assertEquals(stringVariant, validateAndParseVariant(stringVariant));
+    assertEquals(stringVariant, validateAndParseVariant("COL", stringVariant));
 
     // Test custom serializers
     assertEquals(
-        "[-128,0,127]", validateAndParseVariant(new byte[] {Byte.MIN_VALUE, 0, Byte.MAX_VALUE}));
+        "[-128,0,127]",
+        validateAndParseVariant("COL", new byte[] {Byte.MIN_VALUE, 0, Byte.MAX_VALUE}));
     assertEquals(
         "\"2022-09-28T03:04:12.123456789-07:00\"",
         validateAndParseVariant(
+            "COL",
             ZonedDateTime.of(2022, 9, 28, 3, 4, 12, 123456789, ZoneId.of("America/Los_Angeles"))));
 
     // Test forbidden values
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", readTree("{}")));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", new Object()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", "foo"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", new Date()));
     expectError(
         ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseVariant,
-        objectMapper.readTree("{}"));
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseVariant, new Object());
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseVariant, "foo");
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseVariant, new Date());
+        () -> validateAndParseVariant("COL", Collections.singletonList(new Object())));
     expectError(
         ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseVariant,
-        Collections.singletonList(new Object()));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseVariant,
-        Collections.singletonList(Collections.singletonMap("foo", new Object())));
+        () ->
+            validateAndParseVariant(
+                "COL", Collections.singletonList(Collections.singletonMap("foo", new Object()))));
     expectError(
         ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseVariant,
-        Collections.singletonMap(new Object(), "foo"));
+        () -> validateAndParseVariant("COL", Collections.singletonMap(new Object(), "foo")));
     expectError(
         ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseVariant,
-        Collections.singletonMap("foo", new Object()));
+        () -> validateAndParseVariant("COL", Collections.singletonMap("foo", new Object())));
   }
 
   @Test
   public void testValidateAndParseArray() throws Exception {
-    assertEquals("[1]", validateAndParseArray(1));
-    assertEquals("[1]", validateAndParseArray("1"));
-    assertEquals("[1]", validateAndParseArray("                          1   "));
+    assertEquals("[1]", validateAndParseArray("COL", 1));
+    assertEquals("[1]", validateAndParseArray("COL", "1"));
+    assertEquals("[1]", validateAndParseArray("COL", "                          1   "));
     int[] intArray = new int[] {1, 2, 3};
-    assertEquals("[1,2,3]", validateAndParseArray(intArray));
+    assertEquals("[1,2,3]", validateAndParseArray("COL", intArray));
 
     String[] stringArray = new String[] {"a", "b", "c"};
-    assertEquals("[\"a\",\"b\",\"c\"]", validateAndParseArray(stringArray));
+    assertEquals("[\"a\",\"b\",\"c\"]", validateAndParseArray("COL", stringArray));
 
     Object[] objectArray = new Object[] {1, 2, 3};
-    assertEquals("[1,2,3]", validateAndParseArray(objectArray));
+    assertEquals("[1,2,3]", validateAndParseArray("COL", objectArray));
 
     Object[] ObjectArrayWithNull = new Object[] {1, null, 3};
-    assertEquals("[1,null,3]", validateAndParseArray(ObjectArrayWithNull));
+    assertEquals("[1,null,3]", validateAndParseArray("COL", ObjectArrayWithNull));
 
     Object[][] nestedArray = new Object[][] {{1, 2, 3}, null, {4, 5, 6}};
-    assertEquals("[[1,2,3],null,[4,5,6]]", validateAndParseArray(nestedArray));
+    assertEquals("[[1,2,3],null,[4,5,6]]", validateAndParseArray("COL", nestedArray));
 
     List<Integer> intList = Arrays.asList(1, 2, 3);
-    assertEquals("[1,2,3]", validateAndParseArray(intList));
+    assertEquals("[1,2,3]", validateAndParseArray("COL", intList));
 
     List<Object> objectList = Arrays.asList(1, 2, 3);
-    assertEquals("[1,2,3]", validateAndParseArray(objectList));
+    assertEquals("[1,2,3]", validateAndParseArray("COL", objectList));
 
     List<Object> nestedList = Arrays.asList(Arrays.asList(1, 2, 3), 2, 3);
-    assertEquals("[[1,2,3],2,3]", validateAndParseArray(nestedList));
+    assertEquals("[[1,2,3],2,3]", validateAndParseArray("COL", nestedList));
 
     // Test forbidden values
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseArray("COL", readTree("[]")));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseArray("COL", new Object()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseArray("COL", "foo")); // invalid JSO)N
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseArray("COL", new Date()));
     expectError(
         ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseArray,
-        objectMapper.readTree("[]"));
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseArray, new Object());
-    expectError(
-        ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseArray, "foo"); // invalid JSON
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseArray, new Date());
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseArray,
-        Collections.singletonList(new Object()));
+        () -> validateAndParseArray("COL", Collections.singletonList(new Object())));
     expectError(
         ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseArray,
-        Collections.singletonList(Collections.singletonMap("foo", new Object())));
+        () ->
+            validateAndParseArray(
+                "COL", Collections.singletonList(Collections.singletonMap("foo", new Object()))));
     expectError(
         ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseArray,
-        Collections.singletonMap(new Object(), "foo"));
+        () -> validateAndParseArray("COL", Collections.singletonMap(new Object(), "foo")));
     expectError(
         ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseArray,
-        Collections.singletonMap("foo", new Object()));
+        () -> validateAndParseArray("COL", Collections.singletonMap("foo", new Object())));
   }
 
   @Test
   public void testValidateAndParseObject() throws Exception {
     String stringObject = "{\"key\":1}";
-    assertEquals(stringObject, validateAndParseObject(stringObject));
+    assertEquals(stringObject, validateAndParseObject("COL", stringObject));
 
     String badObject = "foo";
     try {
-      validateAndParseObject(badObject);
+      validateAndParseObject("COL", badObject);
       Assert.fail("Expected INVALID_ROW error");
     } catch (SFException err) {
       assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), err.getVendorCode());
@@ -472,41 +511,36 @@ public void testValidateAndParseObject() throws Exception {
     mapVal.put("key", stringVal);
     String tooLargeObject = objectMapper.writeValueAsString(mapVal);
     try {
-      validateAndParseObject(tooLargeObject);
+      validateAndParseObject("COL", tooLargeObject);
       Assert.fail("Expected INVALID_ROW error");
     } catch (SFException err) {
       assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), err.getVendorCode());
     }
 
     // Test forbidden values
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", readTree("{}")));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", "[]"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", "1"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", 1));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", 1.5));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", new Object()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", "foo"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", new Date()));
     expectError(
         ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseObject,
-        objectMapper.readTree("{}"));
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, "[]");
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, "1");
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, 1);
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, 1.5);
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, false);
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, new Object());
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, "foo");
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, new Date());
+        () -> validateAndParseObject("COL", Collections.singletonList(new Object())));
     expectError(
         ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseObject,
-        Collections.singletonList(new Object()));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseObject,
-        Collections.singletonList(Collections.singletonMap("foo", new Object())));
+        () ->
+            validateAndParseObject(
+                "COL", Collections.singletonList(Collections.singletonMap("foo", new Object()))));
     expectError(
         ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseObject,
-        Collections.singletonMap(new Object(), "foo"));
+        () -> validateAndParseObject("COL", Collections.singletonMap(new Object(), "foo")));
     expectError(
         ErrorCode.INVALID_ROW,
-        DataValidationUtil::validateAndParseObject,
-        Collections.singletonMap("foo", new Object()));
+        () -> validateAndParseObject("COL", Collections.singletonMap("foo", new Object())));
   }
 
   @Test
@@ -518,9 +552,9 @@ public void testTooLargeVariant() {
     Map<String, Object> m = new HashMap<>();
     m.put("a", "11");
     m.put("b", new String(stringContent));
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseVariant, m);
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseArray, m);
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseObject, m);
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", m));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseArray("COL", m));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", m));
   }
 
   @Test
@@ -534,21 +568,21 @@ public void testTooLargeMultiByteSemiStructuredValues() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: {a=ČČČČČČČČČČČČČČČČČ.... Value cannot"
-            + " be ingested into Snowflake column VARIANT: Variant too long: length=18874376"
-            + " maxLength=16777152",
-        () -> validateAndParseVariant(m));
+            + " be ingested into Snowflake column COL of type VARIANT: Variant too long:"
+            + " length=18874376 maxLength=16777152",
+        () -> validateAndParseVariant("COL", m));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: [{\"a\":\"ČČČČČČČČČČČČČ.... Value"
-            + " cannot be ingested into Snowflake column ARRAY: Array too large. length=18874378"
-            + " maxLength=16777152",
-        () -> validateAndParseArray(m));
+            + " cannot be ingested into Snowflake column COL of type ARRAY: Array too large."
+            + " length=18874378 maxLength=16777152",
+        () -> validateAndParseArray("COL", m));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: {\"a\":\"ČČČČČČČČČČČČČČ.... Value"
-            + " cannot be ingested into Snowflake column OBJECT: Object too large. length=18874376"
-            + " maxLength=16777152",
-        () -> validateAndParseObject(m));
+            + " cannot be ingested into Snowflake column COL of type OBJECT: Object too large."
+            + " length=18874376 maxLength=16777152",
+        () -> validateAndParseObject("COL", m));
   }
 
   @Test
@@ -709,39 +743,37 @@ public void testValidVariantType() {
 
   @Test
   public void testValidateAndParseDate() {
-    assertEquals(-923, validateAndParseDate("1967-06-23"));
-    assertEquals(-923, validateAndParseDate("1967-06-23 01:01:01"));
-    assertEquals(18464, validateAndParseDate("2020-07-21"));
-    assertEquals(18464, validateAndParseDate("2020-07-21 23:31:00"));
+    assertEquals(-923, validateAndParseDate("COL", "1967-06-23"));
+    assertEquals(-923, validateAndParseDate("COL", "1967-06-23 01:01:01"));
+    assertEquals(18464, validateAndParseDate("COL", "2020-07-21"));
+    assertEquals(18464, validateAndParseDate("COL", "2020-07-21 23:31:00"));
 
     // Time formats are not supported
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("20:57:01.123456789+07:00"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("20:57:01.123456789"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("20:57:01"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("20:57"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("07:57:01.123456789 AM"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("04:01:07 AM"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("04:01 AM"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("04:01 PM"));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "20:57:01.123456789+07:00"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "20:57:01.123456789"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "20:57:01"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "20:57"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "07:57:01.123456789 AM"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "04:01:07 AM"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "04:01 AM"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "04:01 PM"));
 
     // Test forbidden values
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, new Object());
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, LocalTime.now());
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, OffsetTime.now());
-    expectError(
-        ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, new java.util.Date());
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, false);
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, "");
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, "foo");
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, "1.0");
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, 'c');
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, 1);
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, 1L);
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, 1.25);
-    expectError(
-        ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, BigInteger.valueOf(1));
-    expectError(
-        ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseDate, BigDecimal.valueOf(1.25));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", new Object()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", LocalTime.now()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", OffsetTime.now()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", new java.util.Date()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", ""));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "foo"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "1.0"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", 'c'));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", 1));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", 1L));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", 1.25));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", BigInteger.valueOf(1)));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", BigDecimal.valueOf(1.25)));
   }
 
   @Test
@@ -751,68 +783,78 @@ public void testValidateAndParseBinary() {
 
     assertArrayEquals(
         "honk".getBytes(StandardCharsets.UTF_8),
-        validateAndParseBinary("honk".getBytes(StandardCharsets.UTF_8), Optional.empty()));
+        validateAndParseBinary("COL", "honk".getBytes(StandardCharsets.UTF_8), Optional.empty()));
 
     assertArrayEquals(
-        new byte[] {-1, 0, 1}, validateAndParseBinary(new byte[] {-1, 0, 1}, Optional.empty()));
+        new byte[] {-1, 0, 1},
+        validateAndParseBinary("COL", new byte[] {-1, 0, 1}, Optional.empty()));
     assertArrayEquals(
         DatatypeConverter.parseHexBinary(
             "1234567890abcdef"), // pragma: allowlist secret NOT A SECRET
         validateAndParseBinary(
-            "1234567890abcdef", Optional.empty())); // pragma: allowlist secret NOT A SECRET
+            "COL", "1234567890abcdef", Optional.empty())); // pragma: allowlist secret NOT A SECRET
 
-    assertArrayEquals(maxAllowedArray, validateAndParseBinary(maxAllowedArray, Optional.empty()));
     assertArrayEquals(
-        maxAllowedArrayMinusOne, validateAndParseBinary(maxAllowedArrayMinusOne, Optional.empty()));
+        maxAllowedArray, validateAndParseBinary("COL", maxAllowedArray, Optional.empty()));
+    assertArrayEquals(
+        maxAllowedArrayMinusOne,
+        validateAndParseBinary("COL", maxAllowedArrayMinusOne, Optional.empty()));
 
     // Too large arrays should be rejected
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary(new byte[1], Optional.of(0)));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", new byte[1], Optional.of(0)));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseBinary(new byte[BYTES_8_MB + 1], Optional.empty()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary(new byte[8], Optional.of(7)));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("aabb", Optional.of(1)));
+        () -> validateAndParseBinary("COL", new byte[BYTES_8_MB + 1], Optional.empty()));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", new byte[8], Optional.of(7)));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", "aabb", Optional.of(1)));
 
     // unsupported data types should fail
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("000", Optional.empty()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("abcg", Optional.empty()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("c", Optional.empty()));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", "000", Optional.empty()));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", "abcg", Optional.empty()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", "c", Optional.empty()));
     expectError(
         ErrorCode.INVALID_ROW,
         () ->
-            validateAndParseBinary(Arrays.asList((byte) 1, (byte) 2, (byte) 3), Optional.empty()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary(1, Optional.empty()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary(12, Optional.empty()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary(1.5, Optional.empty()));
+            validateAndParseBinary(
+                "COL", Arrays.asList((byte) 1, (byte) 2, (byte) 3), Optional.empty()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", 1, Optional.empty()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", 12, Optional.empty()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", 1.5, Optional.empty()));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseBinary(BigInteger.ONE, Optional.empty()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary(false, Optional.empty()));
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseBinary("COL", BigInteger.ONE, Optional.empty()));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseBinary(new Object(), Optional.empty()));
+        ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", false, Optional.empty()));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", new Object(), Optional.empty()));
   }
 
   @Test
-  public void testValidateAndParseReal() {
+  public void testValidateAndParseReal() throws Exception {
     // From number types
-    assertEquals(1.23d, validateAndParseReal(1.23f), 0);
-    assertEquals(1.23d, validateAndParseReal(1.23), 0);
-    assertEquals(1.23d, validateAndParseReal(1.23d), 0);
-    assertEquals(1.23d, validateAndParseReal(new BigDecimal("1.23")), 0);
+    assertEquals(1.23d, validateAndParseReal("COL", 1.23f), 0);
+    assertEquals(1.23d, validateAndParseReal("COL", 1.23), 0);
+    assertEquals(1.23d, validateAndParseReal("COL", 1.23d), 0);
+    assertEquals(1.23d, validateAndParseReal("COL", new BigDecimal("1.23")), 0);
+    assertEquals(Double.NaN, validateAndParseReal("COL","Nan"), 0);
+    assertEquals(Double.POSITIVE_INFINITY, validateAndParseReal("COL","inF"), 0);
+    assertEquals(Double.NEGATIVE_INFINITY, validateAndParseReal("COL","-inF"), 0);
 
     // From string
-    assertEquals(1.23d, validateAndParseReal("1.23"), 0);
-    assertEquals(123d, validateAndParseReal("1.23E2"), 0);
-    assertEquals(123d, validateAndParseReal("1.23e2"), 0);
-    assertEquals(Double.NaN, validateAndParseReal("Nan"), 0);
-    assertEquals(Double.POSITIVE_INFINITY, validateAndParseReal("inF"), 0);
-    assertEquals(Double.NEGATIVE_INFINITY, validateAndParseReal("-inF"), 0);
+    assertEquals(1.23d, validateAndParseReal("COL", "1.23"), 0);
+    assertEquals(123d, validateAndParseReal("COL", "1.23E2"), 0);
+    assertEquals(123d, validateAndParseReal("COL", "1.23e2"), 0);
 
     // Test forbidden values
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseReal, "foo");
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseReal, 'c');
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseReal, new Object());
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseReal, false);
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseReal, true);
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseReal("COL", "foo"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseReal("COL", 'c'));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseReal("COL", new Object()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseReal("COL", false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseReal("COL", true));
   }
 
   @Test
@@ -821,21 +863,21 @@ public void testValidateAndParseBoolean() {
     for (Object input :
         Arrays.asList(
             true, "true", "True", "TruE", "t", "yes", "YeS", "y", "on", "1", 1.1, -1.1, -10, 10)) {
-      assertEquals(1, validateAndParseBoolean(input));
+      assertEquals(1, validateAndParseBoolean("COL", input));
     }
 
     for (Object input :
         Arrays.asList(false, "false", "False", "FalsE", "f", "no", "NO", "n", "off", "0", 0)) {
-      assertEquals(0, validateAndParseBoolean(input));
+      assertEquals(0, validateAndParseBoolean("COL", input));
     }
 
     // Test forbidden values
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBoolean, new Object());
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBoolean, 't');
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBoolean, 'f');
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBoolean, new int[] {});
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBoolean, "foobar");
-    expectError(ErrorCode.INVALID_ROW, DataValidationUtil::validateAndParseBoolean, "");
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBoolean("COL", new Object()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBoolean("COL", 't'));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBoolean("COL", 'f'));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBoolean("COL", new int[] {}));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBoolean("COL", "foobar"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBoolean("COL", ""));
   }
 
   /**
@@ -848,185 +890,194 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column of type BOOLEAN. Allowed Java types: boolean,"
+            + " be ingested into Snowflake column COL of type BOOLEAN. Allowed Java types: boolean,"
             + " Number, String",
-        () -> validateAndParseBoolean(new Object()));
+        () -> validateAndParseBoolean("COL", new Object()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column BOOLEAN: Not a valid boolean, see"
+            + " Snowflake column COL of type BOOLEAN: Not a valid boolean, see"
             + " https://docs.snowflake.com/en/sql-reference/data-types-logical.html#conversion-to-boolean"
             + " for the list of supported formats",
-        () -> validateAndParseBoolean("abc"));
+        () -> validateAndParseBoolean("COL", "abc"));
 
     // TIME
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column of type TIME. Allowed Java types: String,"
+            + " be ingested into Snowflake column COL of type TIME. Allowed Java types: String,"
             + " LocalTime, OffsetTime",
-        () -> validateAndParseTime(new Object(), 10));
+        () -> validateAndParseTime("COL", new Object(), 10));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column TIME: Not a valid time, see"
+            + " Snowflake column COL of type TIME: Not a valid time, see"
             + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#time-formats"
             + " for the list of supported formats",
-        () -> validateAndParseTime("abc", 10));
+        () -> validateAndParseTime("COL", "abc", 10));
 
     // DATE
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column of type DATE. Allowed Java types: String,"
+            + " be ingested into Snowflake column COL of type DATE. Allowed Java types: String,"
             + " LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
-        () -> validateAndParseDate(new Object()));
+        () -> validateAndParseDate("COL", new Object()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column DATE: Not a valid date, see"
+            + " Snowflake column COL of type DATE: Not a valid date, see"
             + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#date-formats"
             + " for the list of supported formats",
-        () -> validateAndParseDate("abc"));
+        () -> validateAndParseDate("COL", "abc"));
 
     // TIMESTAMP_NTZ
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column of type TIMESTAMP. Allowed Java types: String,"
-            + " LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
-        () -> validateAndParseTimestampNtzSb16(new Object(), 3, true));
+            + " be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
+            + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
+        () -> validateAndParseTimestampNtzSb16("COL", new Object(), 3, true));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column TIMESTAMP: Not a valid timestamp, see"
+            + " Snowflake column COL of type TIMESTAMP: Not a valid timestamp, see"
             + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
             + " for the list of supported formats",
-        () -> validateAndParseTimestampNtzSb16("abc", 3, true));
+        () -> validateAndParseTimestampNtzSb16("COL", "abc", 3, true));
 
     // TIMESTAMP_LTZ
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column of type TIMESTAMP. Allowed Java types: String,"
-            + " LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
-        () -> validateAndParseTimestampNtzSb16(new Object(), 3, false));
+            + " be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
+            + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
+        () -> validateAndParseTimestampNtzSb16("COL", new Object(), 3, false));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column TIMESTAMP: Not a valid timestamp, see"
+            + " Snowflake column COL of type TIMESTAMP: Not a valid timestamp, see"
             + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
             + " for the list of supported formats",
-        () -> validateAndParseTimestampNtzSb16("abc", 3, false));
+        () -> validateAndParseTimestampNtzSb16("COL", "abc", 3, false));
 
     // TIMESTAMP_TZ
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column of type TIMESTAMP. Allowed Java types: String,"
-            + " LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
-        () -> validateAndParseTimestampTz(new Object(), 3));
+            + " be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
+            + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
+        () -> validateAndParseTimestampTz("COL", new Object(), 3));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column TIMESTAMP: Not a valid timestamp, see"
+            + " Snowflake column COL of type TIMESTAMP: Not a valid timestamp, see"
             + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
             + " for the list of supported formats",
-        () -> validateAndParseTimestampTz("abc", 3));
+        () -> validateAndParseTimestampTz("COL", "abc", 3));
 
     // NUMBER
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column of type NUMBER. Allowed Java types: int, long,"
-            + " byte, short, float, double, BigDecimal, BigInteger, String",
-        () -> validateAndParseBigDecimal(new Object()));
+            + " be ingested into Snowflake column COL of type NUMBER. Allowed Java types: int,"
+            + " long, byte, short, float, double, BigDecimal, BigInteger, String",
+        () -> validateAndParseBigDecimal("COL", new Object()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column NUMBER: Not a valid number",
-        () -> validateAndParseBigDecimal("abc"));
+            + " Snowflake column COL of type NUMBER: Not a valid number",
+        () -> validateAndParseBigDecimal("COL", "abc"));
 
     // REAL
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column of type REAL. Allowed Java types: Number, String",
-        () -> validateAndParseReal(new Object()));
+            + " be ingested into Snowflake column COL of type REAL. Allowed Java types: Number,"
+            + " String",
+        () -> validateAndParseReal("COL", new Object()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column REAL: Not a valid decimal number",
-        () -> validateAndParseReal("abc"));
+            + " Snowflake column COL of type REAL: Not a valid decimal number",
+        () -> validateAndParseReal("COL", "abc"));
 
     // STRING
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column of type STRING. Allowed Java types: String,"
+            + " be ingested into Snowflake column COL of type STRING. Allowed Java types: String,"
             + " Number, boolean, char",
-        () -> validateAndParseString(new Object(), Optional.empty()));
+        () -> validateAndParseString("COL", new Object(), Optional.empty()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column STRING: String too long: length=3 maxLength=2",
-        () -> validateAndParseString("abc", Optional.of(2)));
+            + " Snowflake column COL of type STRING: String too long: length=3 maxLength=2",
+        () -> validateAndParseString("COL", "abc", Optional.of(2)));
 
     // BINARY
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column of type BINARY. Allowed Java types: byte[],"
+            + " be ingested into Snowflake column COL of type BINARY. Allowed Java types: byte[],"
             + " String",
-        () -> validateAndParseBinary(new Object(), Optional.empty()));
+        () -> validateAndParseBinary("COL", new Object(), Optional.empty()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: byte[2]. Value cannot be ingested into"
-            + " Snowflake column BINARY: Binary too long: length=2 maxLength=1",
-        () -> validateAndParseBinary(new byte[] {1, 2}, Optional.of(1)));
+            + " Snowflake column COL of type BINARY: Binary too long: length=2 maxLength=1",
+        () -> validateAndParseBinary("COL", new byte[] {1, 2}, Optional.of(1)));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: ghi. Value cannot be ingested into"
-            + " Snowflake column BINARY: Not a valid hex string",
-        () -> validateAndParseBinary("ghi", Optional.empty()));
+            + " Snowflake column COL of type BINARY: Not a valid hex string",
+        () -> validateAndParseBinary("COL", "ghi", Optional.empty()));
 
     // VARIANT
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column of type VARIANT. Allowed Java types: String,"
+            + " be ingested into Snowflake column COL of type VARIANT. Allowed Java types: String,"
             + " Primitive data types and their arrays, java.time.*, List<T>, Map<String, T>, T[]",
-        () -> validateAndParseVariant(new Object()));
+        () -> validateAndParseVariant("COL", new Object()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: ][. Value cannot be ingested into"
-            + " Snowflake column VARIANT: Not a valid JSON",
-        () -> validateAndParseVariant("]["));
+            + " Snowflake column COL of type VARIANT: Not a valid JSON",
+        () -> validateAndParseVariant("COL", "]["));
 
     // ARRAY
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column of type ARRAY. Allowed Java types: String,"
+            + " be ingested into Snowflake column COL of type ARRAY. Allowed Java types: String,"
             + " Primitive data types and their arrays, java.time.*, List<T>, Map<String, T>, T[]",
-        () -> validateAndParseArray(new Object()));
+        () -> validateAndParseArray("COL", new Object()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: ][. Value cannot be ingested into"
-            + " Snowflake column ARRAY: Not a valid JSON",
-        () -> validateAndParseArray("]["));
+            + " Snowflake column COL of type ARRAY: Not a valid JSON",
+        () -> validateAndParseArray("COL", "]["));
 
     // OBJECT
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column of type OBJECT. Allowed Java types: String,"
+            + " be ingested into Snowflake column COL of type OBJECT. Allowed Java types: String,"
             + " Primitive data types and their arrays, java.time.*, List<T>, Map<String, T>, T[]",
-        () -> validateAndParseObject(new Object()));
+        () -> validateAndParseObject("COL", new Object()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: }{. Value cannot be ingested into"
-            + " Snowflake column OBJECT: Not a valid JSON",
-        () -> validateAndParseObject("}{"));
+            + " Snowflake column COL of type OBJECT: Not a valid JSON",
+        () -> validateAndParseObject("COL", "}{"));
+  }
+
+  private JsonNode readTree(String value) {
+    try {
+      return objectMapper.readTree(value);
+    } catch (JsonProcessingException e) {
+      throw new RuntimeException(e);
+    }
   }
 }

From e7544777fad64b49f1cb1c9768a6ac889ae0b082 Mon Sep 17 00:00:00 2001
From: Alkin Sen <alkin.sen@snowflake.com>
Date: Tue, 20 Dec 2022 14:50:46 -0800
Subject: [PATCH 108/356] format.sh run

---
 .../ingest/streaming/internal/DataValidationUtil.java       | 2 +-
 .../ingest/streaming/internal/ParquetValueParser.java       | 2 +-
 .../ingest/streaming/internal/DataValidationUtilTest.java   | 6 +++---
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index b672be213..5eb7a8c39 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -673,7 +673,7 @@ static double validateAndParseReal(String columnName, Object input) {
             return Double.NEGATIVE_INFINITY;
           default:
             throw valueFormatNotAllowedException(
-                    columnName, input, "REAL", "Not a valid decimal number");
+                columnName, input, "REAL", "Not a valid decimal number");
         }
       }
     }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
index cc6130531..5d5acd064 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
@@ -344,7 +344,7 @@ private static byte[] getBinaryValueForLogicalBinary(
     String maxLengthString = columnMetadata.getLength().toString();
     byte[] bytes =
         DataValidationUtil.validateAndParseBinary(
-                columnMetadata.getName(), value, Optional.of(maxLengthString).map(Integer::parseInt));
+            columnMetadata.getName(), value, Optional.of(maxLengthString).map(Integer::parseInt));
 
     String str = new String(bytes, StandardCharsets.UTF_8);
     stats.addStrValue(str);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 1d6827349..ffa0c87df 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -840,9 +840,9 @@ public void testValidateAndParseReal() throws Exception {
     assertEquals(1.23d, validateAndParseReal("COL", 1.23), 0);
     assertEquals(1.23d, validateAndParseReal("COL", 1.23d), 0);
     assertEquals(1.23d, validateAndParseReal("COL", new BigDecimal("1.23")), 0);
-    assertEquals(Double.NaN, validateAndParseReal("COL","Nan"), 0);
-    assertEquals(Double.POSITIVE_INFINITY, validateAndParseReal("COL","inF"), 0);
-    assertEquals(Double.NEGATIVE_INFINITY, validateAndParseReal("COL","-inF"), 0);
+    assertEquals(Double.NaN, validateAndParseReal("COL", "Nan"), 0);
+    assertEquals(Double.POSITIVE_INFINITY, validateAndParseReal("COL", "inF"), 0);
+    assertEquals(Double.NEGATIVE_INFINITY, validateAndParseReal("COL", "-inF"), 0);
 
     // From string
     assertEquals(1.23d, validateAndParseReal("COL", "1.23"), 0);

From a3e605e42b505f1ef1cb9d1abcd8ada10f0afabb Mon Sep 17 00:00:00 2001
From: Alkin Sen <120425561+sfc-gh-asen@users.noreply.github.com>
Date: Mon, 9 Jan 2023 10:46:58 -0800
Subject: [PATCH 109/356] SNOW-711338 Convert empty variant strings into
 Variant column to NULL (#310)

* SNOW-711338 Convert empty variant strings into Variant column to NULL

* Handle null cases properly

* format

* simplified null detection logic and addressed comments

* simplified parquet parser null detection logic

* format run

* added IT for empty string to null conversion

* fix null value variant IT
---
 README.md                                     |   2 +-
 profile.json.example                          |   2 +-
 .../streaming/internal/ArrowRowBuffer.java    |  29 ++--
 .../internal/DataValidationUtil.java          |  10 +-
 .../internal/ParquetValueParser.java          |   9 +-
 .../internal/DataValidationUtilTest.java      |  32 +++-
 .../internal/ParquetValueParserTest.java      |  50 ++++++
 .../streaming/internal/RowBufferTest.java     | 148 ++++++++++++++++++
 .../datatypes/AbstractDataTypeTest.java       |   6 +-
 .../internal/datatypes/SemiStructuredIT.java  |   5 +
 10 files changed, 271 insertions(+), 22 deletions(-)

diff --git a/README.md b/README.md
index 101b9ea77..843bac370 100644
--- a/README.md
+++ b/README.md
@@ -118,7 +118,7 @@ you would need to remove the following scope limits in pom.xml
 -   Use an unencrypted version(Only for testing) of private key while generating keys(private and public pair) using OpenSSL.
 
     -   Here is the link for documentation [Key Pair
-            Generator](https://docs.snowflake.net/manuals/user-guide/python-connector-example.html#using-key-pair-authentication)
+            Generator](https://docs.snowflake.com/en/user-guide/key-pair-auth.html)
 
 # Code style
 
diff --git a/profile.json.example b/profile.json.example
index 4cfb728ba..46dbd42f0 100644
--- a/profile.json.example
+++ b/profile.json.example
@@ -10,6 +10,6 @@
   "database": "database_name",
   "connect_string": "jdbc:snowflake://account_name.snowflakecomputing.com:443",
   "ssl": "on",
-  "warehouse": "warehouse_name"
+  "warehouse": "warehouse_name",
   "role": "role_name"
 }
\ No newline at end of file
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 320558dad..87f1f0bf5 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -455,13 +455,8 @@ private float convertRowToArrow(
       ColumnPhysicalType physicalType =
           ColumnPhysicalType.valueOf(field.getMetadata().get(COLUMN_PHYSICAL_TYPE));
 
-      if (value == null) {
-        if (!field.getFieldType().isNullable()) {
-          throw new SFException(
-              ErrorCode.INVALID_ROW, columnName, "Passed null to non nullable field");
-        }
-        insertNull(vector, stats, curRowIndex);
-      } else {
+      boolean isParsedValueNull = false;
+      if (value != null) {
         switch (logicalType) {
           case FIXED:
             int columnPrecision = Integer.parseInt(field.getMetadata().get(COLUMN_PRECISION));
@@ -543,9 +538,13 @@ private float convertRowToArrow(
             {
               String str =
                   DataValidationUtil.validateAndParseVariant(stats.getColumnDisplayName(), value);
-              Text text = new Text(str);
-              ((VarCharVector) vector).setSafe(curRowIndex, text);
-              rowBufferSize += text.getBytes().length;
+              if (str != null) {
+                Text text = new Text(str);
+                ((VarCharVector) vector).setSafe(curRowIndex, text);
+                rowBufferSize += text.getBytes().length;
+              } else {
+                isParsedValueNull = true;
+              }
               break;
             }
           case TIMESTAMP_LTZ:
@@ -696,7 +695,6 @@ private float convertRowToArrow(
                   BigInteger timeInScale =
                       DataValidationUtil.validateAndParseTime(
                           stats.getColumnDisplayName(), value, getColumnScale(field.getMetadata()));
-                  stats.addIntValue(timeInScale);
                   ((IntVector) vector).setSafe(curRowIndex, timeInScale.intValue());
                   stats.addIntValue(timeInScale);
                   rowBufferSize += 4;
@@ -747,6 +745,15 @@ private float convertRowToArrow(
             throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
         }
       }
+
+      if (value == null || isParsedValueNull) {
+        if (!field.getFieldType().isNullable()) {
+          throw new SFException(
+              ErrorCode.INVALID_ROW, columnName, "Passed null to non nullable field");
+        } else {
+          insertNull(vector, stats, curRowIndex);
+        }
+      }
     }
 
     // Insert nulls to the columns that doesn't show up in the input
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 5eb7a8c39..778a9ed77 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -120,8 +120,14 @@ private static JsonNode validateAndParseSemiStructuredAsJsonTree(
    * @return JSON string representing the input
    */
   static String validateAndParseVariant(String columnName, Object input) {
-    String output =
-        validateAndParseSemiStructuredAsJsonTree(columnName, input, "VARIANT").toString();
+    JsonNode node = validateAndParseSemiStructuredAsJsonTree(columnName, input, "VARIANT");
+
+    // Missing nodes are not valid json, ingest them as NULL instead
+    if (node.isMissingNode()) {
+      return null;
+    }
+
+    String output = node.toString();
     int stringLength = output.getBytes(StandardCharsets.UTF_8).length;
     if (stringLength > MAX_SEMI_STRUCTURED_LENGTH) {
       throw valueFormatNotAllowedException(
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
index 5d5acd064..4462369c8 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
@@ -105,7 +105,9 @@ static ParquetBufferValue parseColumnValueToParquet(
           } else {
             String str = getBinaryValue(value, stats, columnMetadata);
             value = str;
-            size = str.getBytes().length;
+            if (str != null) {
+              size = str.getBytes().length;
+            }
           }
           break;
         case FIXED_LEN_BYTE_ARRAY:
@@ -124,13 +126,16 @@ static ParquetBufferValue parseColumnValueToParquet(
         default:
           throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
       }
-    } else {
+    }
+
+    if (value == null) {
       if (!columnMetadata.getNullable()) {
         throw new SFException(
             ErrorCode.INVALID_ROW, columnMetadata.getName(), "Passed null to non nullable field");
       }
       stats.incCurrentNullCount();
     }
+
     return new ParquetBufferValue(value, size);
   }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index ffa0c87df..d4caf2e16 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -17,6 +17,7 @@
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseVariant;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -54,8 +55,7 @@ private void expectErrorCodeAndMessage(
       Assert.fail("Expected Exception");
     } catch (SFException e) {
       assertEquals(expectedErrorCode.getMessageCode(), e.getVendorCode());
-      if (expectedExceptionMessage != null)
-        Assert.assertEquals(expectedExceptionMessage, e.getMessage());
+      if (expectedExceptionMessage != null) assertEquals(expectedExceptionMessage, e.getMessage());
     } catch (Exception e) {
       Assert.fail("Invalid error through");
     }
@@ -373,12 +373,12 @@ public void testValidateAndParseString() {
       longBuilder.append("č"); // max string length is measured in chars, not bytes
     }
     String maxString = longBuilder.toString();
-    Assert.assertEquals(maxString, validateAndParseString("COL", maxString, Optional.empty()));
+    assertEquals(maxString, validateAndParseString("COL", maxString, Optional.empty()));
 
     // max length - 1 should also succeed
     longBuilder.setLength(BYTES_16_MB - 1);
     String maxStringMinusOne = longBuilder.toString();
-    Assert.assertEquals(
+    assertEquals(
         maxStringMinusOne, validateAndParseString("COL", maxStringMinusOne, Optional.empty()));
 
     // max length + 1 should fail
@@ -420,7 +420,25 @@ public void testValidateAndParseVariant() throws Exception {
             "COL",
             ZonedDateTime.of(2022, 9, 28, 3, 4, 12, 123456789, ZoneId.of("America/Los_Angeles"))));
 
+    // Test valid JSON tokens
+    assertEquals("null", validateAndParseVariant("COL", null));
+    assertEquals("null", validateAndParseVariant("COL", "null"));
+    assertEquals("true", validateAndParseVariant("COL", true));
+    assertEquals("true", validateAndParseVariant("COL", "true"));
+    assertEquals("false", validateAndParseVariant("COL", false));
+    assertEquals("false", validateAndParseVariant("COL", "false"));
+    assertEquals("{}", validateAndParseVariant("COL", "{}"));
+    assertEquals("[]", validateAndParseVariant("COL", "[]"));
+    assertEquals("[\"foo\",1,null]", validateAndParseVariant("COL", "[\"foo\",1,null]"));
+    assertEquals("\"\"", validateAndParseVariant("COL", "\"\""));
+
+    // Test missing values are null instead of empty string
+    assertNull(validateAndParseVariant("COL", ""));
+    assertNull(validateAndParseVariant("COL", "  "));
+
     // Test forbidden values
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", "{null}"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", "}{"));
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", readTree("{}")));
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", new Object()));
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", "foo"));
@@ -470,6 +488,12 @@ public void testValidateAndParseArray() throws Exception {
     List<Object> nestedList = Arrays.asList(Arrays.asList(1, 2, 3), 2, 3);
     assertEquals("[[1,2,3],2,3]", validateAndParseArray("COL", nestedList));
 
+    // Test null values
+    assertEquals("[null]", validateAndParseArray("COL", ""));
+    assertEquals("[null]", validateAndParseArray("COL", " "));
+    assertEquals("[null]", validateAndParseArray("COL", "null"));
+    assertEquals("[null]", validateAndParseArray("COL", null));
+
     // Test forbidden values
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseArray("COL", readTree("[]")));
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseArray("COL", new Object()));
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
index e580cc9b6..459b4f5cd 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
@@ -283,6 +283,45 @@ private void testJsonWithLogicalType(String logicalType) {
         .assertMatches();
   }
 
+  @Test
+  public void parseValueNullVariantToBinary() {
+    testNullJsonWithLogicalType(null);
+  }
+
+  @Test
+  public void parseValueEmptyStringVariantToBinary() {
+    testNullJsonWithLogicalType("");
+  }
+
+  @Test
+  public void parseValueEmptySpaceStringVariantToBinary() {
+    testNullJsonWithLogicalType("     ");
+  }
+
+  private void testNullJsonWithLogicalType(String var) {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("VARIANT")
+            .physicalType("BINARY")
+            .nullable(true)
+            .build();
+
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            var, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(String.class)
+        .expectedParsedValue(var)
+        .expectedSize(0)
+        .expectedMinMax(null)
+        .expectedNullCount(1)
+        .assertNull();
+  }
+
   @Test
   public void parseValueArrayToBinary() {
     ColumnMetadata testCol =
@@ -504,6 +543,7 @@ private static class ParquetValueParserAssertionBuilder {
     private Object value;
     private float size;
     private Object minMaxStat;
+    private long currentNullCount;
 
     static ParquetValueParserAssertionBuilder newBuilder() {
       ParquetValueParserAssertionBuilder builder = new ParquetValueParserAssertionBuilder();
@@ -541,6 +581,11 @@ public ParquetValueParserAssertionBuilder expectedMinMax(Object minMaxStat) {
       return this;
     }
 
+    public ParquetValueParserAssertionBuilder expectedNullCount(long currentNullCount) {
+      this.currentNullCount = currentNullCount;
+      return this;
+    }
+
     void assertMatches() {
       Assert.assertEquals(valueClass, parquetBufferValue.getValue().getClass());
       if (valueClass.equals(byte[].class)) {
@@ -565,5 +610,10 @@ void assertMatches() {
       }
       throw new IllegalArgumentException("Unknown data type for min stat");
     }
+
+    void assertNull() {
+      Assert.assertNull(parquetBufferValue.getValue());
+      Assert.assertEquals(currentNullCount, rowBufferStats.getCurrentNullCount());
+    }
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index b5ed831b4..6d25b11b3 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -328,6 +328,26 @@ private void testInsertRowHelper(AbstractRowBuffer<?> rowBuffer) {
     Assert.assertFalse(response.hasErrors());
   }
 
+  @Test
+  public void testNullInsertRow() {
+    testInsertNullRowHelper(this.rowBufferOnErrorContinue);
+    testInsertNullRowHelper(this.rowBufferOnErrorAbort);
+  }
+
+  private void testInsertNullRowHelper(AbstractRowBuffer<?> rowBuffer) {
+    Map<String, Object> row = new HashMap<>();
+    row.put("colTinyInt", null);
+    row.put("\"colTinyInt\"", null);
+    row.put("colSmallInt", null);
+    row.put("colInt", null);
+    row.put("colBigInt", null);
+    row.put("colDecimal", null);
+    row.put("colChar", null);
+
+    InsertValidationResponse response = rowBuffer.insertRows(Collections.singletonList(row), null);
+    Assert.assertFalse(response.hasErrors());
+  }
+
   @Test
   public void testInsertRows() {
     testInsertRowsHelper(this.rowBufferOnErrorContinue);
@@ -1201,4 +1221,132 @@ public void testOnErrorAbortFailures() {
     Assert.assertEquals(3, data.getRowCount());
     Assert.assertEquals(0, innerBuffer.rowCount);
   }
+
+  @Test
+  public void testE2EVariant() {
+    testE2EVariantHelper(OpenChannelRequest.OnErrorOption.ABORT);
+    testE2EVariantHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+  }
+
+  private void testE2EVariantHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
+    AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
+
+    ColumnMetadata colVariant = new ColumnMetadata();
+    colVariant.setName("COLVARIANT");
+    colVariant.setPhysicalType("LOB");
+    colVariant.setNullable(true);
+    colVariant.setLogicalType("VARIANT");
+
+    innerBuffer.setupSchema(Collections.singletonList(colVariant));
+
+    Map<String, Object> row1 = new HashMap<>();
+    row1.put("COLVARIANT", null);
+
+    Map<String, Object> row2 = new HashMap<>();
+    row2.put("COLVARIANT", "");
+
+    Map<String, Object> row3 = new HashMap<>();
+    row3.put("COLVARIANT", "null");
+
+    Map<String, Object> row4 = new HashMap<>();
+    row4.put("COLVARIANT", "{\"key\":1}");
+
+    Map<String, Object> row5 = new HashMap<>();
+    row5.put("COLVARIANT", 3);
+
+    InsertValidationResponse response =
+        innerBuffer.insertRows(Arrays.asList(row1, row2, row3, row4, row5), null);
+    Assert.assertFalse(response.hasErrors());
+
+    // Check data was inserted into the buffer correctly
+    Assert.assertNull(null, innerBuffer.getVectorValueAt("COLVARIANT", 0));
+    Assert.assertNull(null, innerBuffer.getVectorValueAt("COLVARIANT", 1));
+    Assert.assertEquals("null", innerBuffer.getVectorValueAt("COLVARIANT", 2));
+    Assert.assertEquals("{\"key\":1}", innerBuffer.getVectorValueAt("COLVARIANT", 3));
+    Assert.assertEquals("3", innerBuffer.getVectorValueAt("COLVARIANT", 4));
+
+    // Check stats generation
+    ChannelData<?> result = innerBuffer.flush("my_snowpipe_streaming.bdec");
+    Assert.assertEquals(5, result.getRowCount());
+    Assert.assertEquals(2, result.getColumnEps().get("COLVARIANT").getCurrentNullCount());
+  }
+
+  @Test
+  public void testE2EObject() {
+    testE2EObjectHelper(OpenChannelRequest.OnErrorOption.ABORT);
+    testE2EObjectHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+  }
+
+  private void testE2EObjectHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
+    AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
+
+    ColumnMetadata colObject = new ColumnMetadata();
+    colObject.setName("COLOBJECT");
+    colObject.setPhysicalType("LOB");
+    colObject.setNullable(true);
+    colObject.setLogicalType("OBJECT");
+
+    innerBuffer.setupSchema(Collections.singletonList(colObject));
+
+    Map<String, Object> row1 = new HashMap<>();
+    row1.put("COLOBJECT", "{\"key\":1}");
+
+    InsertValidationResponse response = innerBuffer.insertRows(Arrays.asList(row1), null);
+    Assert.assertFalse(response.hasErrors());
+
+    // Check data was inserted into the buffer correctly
+    Assert.assertEquals("{\"key\":1}", innerBuffer.getVectorValueAt("COLOBJECT", 0));
+
+    // Check stats generation
+    ChannelData<?> result = innerBuffer.flush("my_snowpipe_streaming.bdec");
+    Assert.assertEquals(1, result.getRowCount());
+  }
+
+  @Test
+  public void testE2EArray() {
+    testE2EArrayHelper(OpenChannelRequest.OnErrorOption.ABORT);
+    testE2EArrayHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+  }
+
+  private void testE2EArrayHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
+    AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
+
+    ColumnMetadata colObject = new ColumnMetadata();
+    colObject.setName("COLARRAY");
+    colObject.setPhysicalType("LOB");
+    colObject.setNullable(true);
+    colObject.setLogicalType("ARRAY");
+
+    innerBuffer.setupSchema(Collections.singletonList(colObject));
+
+    Map<String, Object> row1 = new HashMap<>();
+    row1.put("COLARRAY", null);
+
+    Map<String, Object> row2 = new HashMap<>();
+    row2.put("COLARRAY", "");
+
+    Map<String, Object> row3 = new HashMap<>();
+    row3.put("COLARRAY", "null");
+
+    Map<String, Object> row4 = new HashMap<>();
+    row4.put("COLARRAY", "{\"key\":1}");
+
+    Map<String, Object> row5 = new HashMap<>();
+    row5.put("COLARRAY", Arrays.asList(1, 2, 3));
+
+    InsertValidationResponse response =
+        innerBuffer.insertRows(Arrays.asList(row1, row2, row3, row4, row5), null);
+    Assert.assertFalse(response.hasErrors());
+
+    // Check data was inserted into the buffer correctly
+    Assert.assertNull(innerBuffer.getVectorValueAt("COLARRAY", 0));
+    Assert.assertEquals("[null]", innerBuffer.getVectorValueAt("COLARRAY", 1));
+    Assert.assertEquals("[null]", innerBuffer.getVectorValueAt("COLARRAY", 2));
+    Assert.assertEquals("[{\"key\":1}]", innerBuffer.getVectorValueAt("COLARRAY", 3));
+    Assert.assertEquals("[1,2,3]", innerBuffer.getVectorValueAt("COLARRAY", 4));
+
+    // Check stats generation
+    ChannelData<?> result = innerBuffer.flush("my_snowpipe_streaming.bdec");
+    Assert.assertEquals(5, result.getRowCount());
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index ae453da31..f6de25e31 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -336,7 +336,11 @@ <STREAMING_INGEST_WRITE> void assertVariant(
     }
 
     Assert.assertEquals(1, counter);
-    Assert.assertEquals(objectMapper.readTree(expectedValue), objectMapper.readTree(value));
+    if (expectedValue == null) {
+      Assert.assertNull(value);
+    } else {
+      Assert.assertEquals(objectMapper.readTree(expectedValue), objectMapper.readTree(value));
+    }
     Assert.assertEquals(expectedType, typeof);
     migrateTable(tableName); // migration should always succeed
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
index 1195d2bfc..bf55b3396 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
@@ -169,6 +169,11 @@ public void testVariant() throws Exception {
 
     // Test JSON null
     assertVariant("VARIANT", "null", "null", "NULL_VALUE");
+
+    // Test SQL null, if the value is SQL NULL, the value returned is null
+    assertVariant("VARIANT", "", null, null);
+    assertVariant("VARIANT", " ", null, null);
+    assertVariant("VARIANT", null, null, null);
   }
 
   @Test

From bb76fc3d996049db3ed0a68e28c9e2c62fbfe776 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Tue, 10 Jan 2023 14:12:26 +0100
Subject: [PATCH 110/356] SNOW-710401 Avoid datetime parsing wherever possible
 (#311)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This patch implements more efficient calculation of `SFTimestamp`
from supported `java.time.*` objects, without calling `toString`
on them and parsing the result.

Users should be advised to pass date/time values either as
unix timestamp strings or as `java.time.*` objects. Passing dates
as formatted strings requires parsing, which is less efficient.

A simple microbenchmark on my local machine shows ≈19x improvement.
Sequentially parsing and validating  10M ZonedDateTime objects took
≈36s with the old approach compared to 1.9s with the new approach.

Additionally, this commit unifies processing of timestamps
with and without timezones.
---
 .../streaming/internal/ArrowRowBuffer.java    |  18 +-
 .../internal/DataValidationUtil.java          | 219 +++++++++---------
 .../internal/ParquetValueParser.java          |   9 +-
 .../internal/DataValidationUtilTest.java      | 153 ++++++------
 .../internal/datatypes/DateTimeIT.java        |  42 +++-
 5 files changed, 237 insertions(+), 204 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 87f1f0bf5..af385ed26 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -556,7 +556,7 @@ private float convertRowToArrow(
                 {
                   BigIntVector bigIntVector = (BigIntVector) vector;
                   TimestampWrapper timestampWrapper =
-                      DataValidationUtil.validateAndParseTimestampNtzSb16(
+                      DataValidationUtil.validateAndParseTimestamp(
                           stats.getColumnDisplayName(),
                           value,
                           getColumnScale(field.getMetadata()),
@@ -577,7 +577,7 @@ private float convertRowToArrow(
                   structVector.setIndexDefined(curRowIndex);
 
                   TimestampWrapper timestampWrapper =
-                      DataValidationUtil.validateAndParseTimestampNtzSb16(
+                      DataValidationUtil.validateAndParseTimestamp(
                           stats.getColumnDisplayName(),
                           value,
                           getColumnScale(field.getMetadata()),
@@ -605,8 +605,11 @@ private float convertRowToArrow(
                   structVector.setIndexDefined(curRowIndex);
 
                   TimestampWrapper timestampWrapper =
-                      DataValidationUtil.validateAndParseTimestampTz(
-                          stats.getColumnDisplayName(), value, getColumnScale(field.getMetadata()));
+                      DataValidationUtil.validateAndParseTimestamp(
+                          stats.getColumnDisplayName(),
+                          value,
+                          getColumnScale(field.getMetadata()),
+                          false);
                   epochVector.setSafe(curRowIndex, timestampWrapper.getTimeInScale().longValue());
                   timezoneVector.setSafe(
                       curRowIndex,
@@ -645,8 +648,11 @@ private float convertRowToArrow(
                   structVector.setIndexDefined(curRowIndex);
 
                   TimestampWrapper timestampWrapper =
-                      DataValidationUtil.validateAndParseTimestampTz(
-                          stats.getColumnDisplayName(), value, getColumnScale(field.getMetadata()));
+                      DataValidationUtil.validateAndParseTimestamp(
+                          stats.getColumnDisplayName(),
+                          value,
+                          getColumnScale(field.getMetadata()),
+                          false);
                   epochVector.setSafe(curRowIndex, timestampWrapper.getEpoch());
                   fractionVector.setSafe(curRowIndex, timestampWrapper.getFraction());
                   timezoneVector.setSafe(
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 778a9ed77..09e459de6 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -15,18 +15,19 @@
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
+import java.time.Instant;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.LocalTime;
 import java.time.OffsetDateTime;
 import java.time.OffsetTime;
 import java.time.ZonedDateTime;
-import java.time.format.DateTimeFormatter;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.SimpleTimeZone;
 import java.util.TimeZone;
 import java.util.concurrent.TimeUnit;
 import javax.xml.bind.DatatypeConverter;
@@ -303,8 +304,8 @@ static String validateAndParseObject(String columnName, Object input) {
   }
 
   /**
-   * Validates and parses input for TIMESTAMP_NTZ or TIMESTAMP_LTZ Snowflake type. Allowed Java
-   * types:
+   * Validates and parses input for TIMESTAMP_NTZ, TIMESTAMP_LTZ and TIMEATAMP_TZ Snowflake types.
+   * Allowed Java types:
    *
    * <ul>
    *   <li>String
@@ -314,41 +315,38 @@ static String validateAndParseObject(String columnName, Object input) {
    *   <li>ZonedDateTime
    * </ul>
    *
-   * @param input String date in valid format or seconds past the epoch. Accepts fractional seconds
-   *     with precision up to the column's scale
+   * @param input String date in valid format, seconds past the epoch or java.time.* object. Accepts
+   *     fractional seconds with precision up to the column's scale
    * @param scale decimal scale of timestamp 16 byte integer
    * @param ignoreTimezone Must be true for TIMESTAMP_NTZ
    * @return TimestampWrapper with epoch seconds, fractional seconds, and epoch time in the column
    *     scale
    */
-  static TimestampWrapper validateAndParseTimestampNtzSb16(
+  static TimestampWrapper validateAndParseTimestamp(
       String columnName, Object input, int scale, boolean ignoreTimezone) {
-    String valueString;
-    if (input instanceof String) valueString = (String) input;
-    else if (input instanceof LocalDate) valueString = input.toString();
-    else if (input instanceof LocalDateTime) valueString = input.toString();
-    else if (input instanceof ZonedDateTime)
-      valueString =
-          ignoreTimezone
-              ? ((ZonedDateTime) input).toLocalDateTime().toString()
-              : DateTimeFormatter.ISO_DATE_TIME.format(((ZonedDateTime) input).toOffsetDateTime());
-    else if (input instanceof OffsetDateTime)
-      valueString =
-          ignoreTimezone
-              ? ((OffsetDateTime) input).toLocalDateTime().toString()
-              : DateTimeFormatter.ISO_DATE_TIME.format((OffsetDateTime) input);
-    else
+    TimeZone effectiveTimeZone = ignoreTimezone ? GMT : DEFAULT_TIMEZONE;
+    SFTimestamp timestamp;
+    if (input instanceof String) {
+      SnowflakeDateTimeFormat snowflakeDateTimeFormatter = createDateTimeFormatter();
+      timestamp =
+          snowflakeDateTimeFormatter.parse(
+              (String) input, effectiveTimeZone, 0, DATE | TIMESTAMP, ignoreTimezone, null);
+    } else if (input instanceof LocalDate) {
+      timestamp = timeStampFromLocalDate((LocalDate) input, effectiveTimeZone);
+    } else if (input instanceof LocalDateTime) {
+      timestamp = timeStampFromLocalDateTime((LocalDateTime) input, effectiveTimeZone);
+    } else if (input instanceof ZonedDateTime) {
+      timestamp = timestampFromZonedDateTime((ZonedDateTime) input, ignoreTimezone);
+    } else if (input instanceof OffsetDateTime) {
+      timestamp = timestampFromOffsetDateTime((OffsetDateTime) input, ignoreTimezone);
+    } else {
       throw typeNotAllowedException(
           columnName,
           input.getClass(),
           "TIMESTAMP",
           new String[] {"String", "LocalDate", "LocalDateTime", "ZonedDateTime", "OffsetDateTime"});
+    }
 
-    SnowflakeDateTimeFormat snowflakeDateTimeFormatter = createDateTimeFormatter();
-    TimeZone effectiveTimeZone = ignoreTimezone ? GMT : DEFAULT_TIMEZONE;
-    SFTimestamp timestamp =
-        snowflakeDateTimeFormatter.parse(
-            valueString, effectiveTimeZone, 0, DATE | TIMESTAMP, ignoreTimezone, null);
     if (timestamp != null) {
       long epoch = timestamp.getSeconds().longValue();
       int fractionInScale = getFractionFromTimestamp(timestamp) / Power10.intTable[9 - scale];
@@ -357,16 +355,16 @@ else if (input instanceof OffsetDateTime)
               .multiply(Power10.sb16Table[scale])
               .add(BigInteger.valueOf(fractionInScale));
       return new TimestampWrapper(
-          epoch, fractionInScale * Power10.intTable[9 - scale], timeInScale);
-    } else {
-      throw valueFormatNotAllowedException(
-          columnName,
-          input.toString(),
-          "TIMESTAMP",
-          "Not a valid timestamp, see"
-              + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
-              + " for the list of supported formats");
+          epoch, fractionInScale * Power10.intTable[9 - scale], timeInScale, timestamp);
     }
+
+    throw valueFormatNotAllowedException(
+        columnName,
+        input.toString(),
+        "TIMESTAMP",
+        "Not a valid timestamp, see"
+            + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
+            + " for the list of supported formats");
   }
 
   /**
@@ -382,64 +380,6 @@ private static int getFractionFromTimestamp(SFTimestamp input) {
     return input.getNanosSinceEpoch().subtract(epochSecondsInNanoSeconds).intValue();
   }
 
-  /**
-   * Validates and parses input for TIMESTAMP_TZ Snowflake type. Allowed Java types:
-   *
-   * <ul>
-   *   <li>String
-   *   <li>LocalDate
-   *   <li>LocalDateTime
-   *   <li>OffsetDateTime
-   *   <li>ZonedDateTime
-   * </ul>
-   *
-   * @param input TIMESTAMP_TZ in "2021-01-01 01:00:00 +0100" format
-   * @param scale decimal scale of timestamp 16 byte integer
-   * @return TimestampWrapper with epoch seconds, fractional seconds, and epoch time in the column
-   *     scale
-   */
-  static TimestampWrapper validateAndParseTimestampTz(String columnName, Object input, int scale) {
-
-    String stringInput;
-    if (input instanceof String) stringInput = (String) input;
-    else if (input instanceof LocalDate) stringInput = input.toString();
-    else if (input instanceof LocalDateTime) stringInput = input.toString();
-    else if (input instanceof ZonedDateTime)
-      stringInput =
-          DateTimeFormatter.ISO_DATE_TIME.format(((ZonedDateTime) input).toOffsetDateTime());
-    else if (input instanceof OffsetDateTime)
-      stringInput = DateTimeFormatter.ISO_DATE_TIME.format((OffsetDateTime) input);
-    else
-      throw typeNotAllowedException(
-          columnName,
-          input.getClass(),
-          "TIMESTAMP",
-          new String[] {"String", "LocalDate", "LocalDateTime", "ZonedDateTime", "OffsetDateTime"});
-
-    SnowflakeDateTimeFormat snowflakeDateTimeFormatter = createDateTimeFormatter();
-
-    SFTimestamp timestamp =
-        snowflakeDateTimeFormatter.parse(
-            stringInput, DEFAULT_TIMEZONE, 0, DATE | TIMESTAMP, false, null);
-    if (timestamp != null) {
-      long epoch = timestamp.getSeconds().longValue();
-      int fractionInScale = getFractionFromTimestamp(timestamp) / Power10.intTable[9 - scale];
-      return new TimestampWrapper(
-          epoch,
-          fractionInScale * Power10.intTable[9 - scale],
-          BigInteger.valueOf(epoch * Power10.intTable[scale] + fractionInScale),
-          timestamp);
-    } else {
-      throw valueFormatNotAllowedException(
-          columnName,
-          input.toString(),
-          "TIMESTAMP",
-          "Not a valid timestamp, see"
-              + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
-              + " for the list of supported formats");
-    }
-  }
-
   /**
    * Converts input to string, validates that length is less than max allowed string size
    * https://docs.snowflake.com/en/sql-reference/data-types-text.html#varchar. Allowed data types:
@@ -483,6 +423,7 @@ static String validateAndParseString(
     }
     return output;
   }
+
   /**
    * Returns a BigDecimal representation of the input. Strings of the form "1.23E4" will be treated
    * as being written in * scientific notation (e.g. 1.23 * 10^4). Does not perform any size
@@ -533,17 +474,18 @@ static BigDecimal validateAndParseBigDecimal(String columnName, Object input) {
    * </ul>
    */
   static int validateAndParseDate(String columnName, Object input) {
-    String inputString;
+    SFTimestamp timestamp;
     if (input instanceof String) {
-      inputString = (String) input;
+      timestamp =
+          createDateTimeFormatter().parse((String) input, GMT, 0, DATE | TIMESTAMP, true, null);
     } else if (input instanceof LocalDate) {
-      inputString = input.toString();
+      timestamp = timeStampFromLocalDate((LocalDate) input, GMT);
     } else if (input instanceof LocalDateTime) {
-      inputString = ((LocalDateTime) input).toLocalDate().toString();
+      timestamp = timeStampFromLocalDateTime((LocalDateTime) input, GMT);
     } else if (input instanceof ZonedDateTime) {
-      inputString = ((ZonedDateTime) input).toLocalDate().toString();
+      timestamp = timestampFromZonedDateTime((ZonedDateTime) input, true);
     } else if (input instanceof OffsetDateTime) {
-      inputString = ((OffsetDateTime) input).toLocalDate().toString();
+      timestamp = timestampFromOffsetDateTime((OffsetDateTime) input, true);
     } else {
       throw typeNotAllowedException(
           columnName,
@@ -552,8 +494,6 @@ static int validateAndParseDate(String columnName, Object input) {
           new String[] {"String", "LocalDate", "LocalDateTime", "ZonedDateTime", "OffsetDateTime"});
     }
 
-    SFTimestamp timestamp =
-        createDateTimeFormatter().parse(inputString, GMT, 0, DATE | TIMESTAMP, true, null);
     if (timestamp == null)
       throw valueFormatNotAllowedException(
           columnName,
@@ -617,21 +557,24 @@ static byte[] validateAndParseBinary(
    * </ul>
    */
   static BigInteger validateAndParseTime(String columnName, Object input, int scale) {
-    String stringInput;
+    SFTimestamp timestamp;
     if (input instanceof String) {
-      stringInput = (String) input;
+      String stringInput = (String) input;
+      timestamp =
+          createDateTimeFormatter()
+              .parse(stringInput, GMT, 0, SnowflakeDateTimeFormat.TIME, true, null);
     } else if (input instanceof LocalTime) {
-      stringInput = input.toString();
+      timestamp =
+          timeStampFromLocalDateTime(((LocalTime) input).atDate(LocalDate.ofEpochDay(0)), GMT);
     } else if (input instanceof OffsetTime) {
-      stringInput = ((OffsetTime) input).toLocalTime().toString();
+      timestamp =
+          timeStampFromLocalDateTime(
+              ((OffsetTime) input).toLocalTime().atDate(LocalDate.ofEpochDay(0)), GMT);
     } else {
       throw typeNotAllowedException(
           columnName, input.getClass(), "TIME", new String[] {"String", "LocalTime", "OffsetTime"});
     }
 
-    SFTimestamp timestamp =
-        createDateTimeFormatter()
-            .parse(stringInput, GMT, 0, SnowflakeDateTimeFormat.TIME, true, null);
     if (timestamp == null) {
       throw valueFormatNotAllowedException(
           columnName,
@@ -640,13 +583,13 @@ static BigInteger validateAndParseTime(String columnName, Object input, int scal
           "Not a valid time, see"
               + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#time-formats"
               + " for the list of supported formats");
-    } else {
-      return timestamp
-          .getNanosSinceEpoch()
-          .divide(BigDecimal.valueOf(Power10.intTable[9 - scale]))
-          .toBigInteger()
-          .mod(BigInteger.valueOf(24L * 60 * 60 * Power10.intTable[scale]));
     }
+
+    return timestamp
+        .getNanosSinceEpoch()
+        .toBigInteger()
+        .divide(Power10.sb16Table[9 - scale])
+        .mod(BigInteger.valueOf(24L * 60 * 60).multiply(Power10.sb16Table[scale]));
   }
 
   /**
@@ -793,4 +736,52 @@ private static String sanitizeValueForExceptionMessage(Object value) {
     String valueString = value.toString();
     return valueString.length() <= maxSize ? valueString : valueString.substring(0, 20) + "...";
   }
+
+  /**
+   * Constructs SFTimestamp from {@link LocalDate}. Default timezone is used for _TZ and _LTZ and
+   * UTC for _NTZ.
+   */
+  private static SFTimestamp timeStampFromLocalDate(LocalDate date, TimeZone tz) {
+    return timeStampFromLocalDateTime(date.atStartOfDay(), tz);
+  }
+
+  /**
+   * Constructs SFTimestamp from {@link LocalDateTime}. Default timezone is used for _TZ and _LTZ
+   * and UTC for _NTZ.
+   */
+  private static SFTimestamp timeStampFromLocalDateTime(LocalDateTime localDateTime, TimeZone tz) {
+    return timestampFromInstant(localDateTime.atZone(tz.toZoneId()).toInstant(), tz);
+  }
+
+  /** Constructs SFTimestamp from {@link ZonedDateTime}. Timezone is dropped for _NTZ. */
+  private static SFTimestamp timestampFromZonedDateTime(
+      ZonedDateTime zonedDateTime, boolean ignoreTimezone) {
+    if (ignoreTimezone) {
+      LocalDateTime local = zonedDateTime.toLocalDateTime();
+      return timeStampFromLocalDateTime(local, GMT);
+    }
+
+    TimeZone timeZone = TimeZone.getTimeZone(zonedDateTime.getZone().toString());
+    return timestampFromInstant(zonedDateTime.toInstant(), timeZone);
+  }
+
+  /** Constructs SFTimestamp from {@link OffsetDateTime}. Timezone is dropped for _NTZ. */
+  private static SFTimestamp timestampFromOffsetDateTime(
+      OffsetDateTime offsetDateTime, boolean dropTimezone) {
+    if (dropTimezone) {
+      LocalDateTime local = offsetDateTime.toLocalDateTime();
+      return timeStampFromLocalDateTime(local, GMT);
+    }
+    TimeZone tz =
+        new SimpleTimeZone(
+            offsetDateTime.getOffset().getTotalSeconds() * 1000,
+            "GENERATED_SNOWPIPE_STREAMING:" + offsetDateTime.getOffset());
+    return timestampFromInstant(offsetDateTime.toInstant(), tz);
+  }
+
+  /** Constructs SFTimestamp from {@link Instant} and time zone. */
+  private static SFTimestamp timestampFromInstant(Instant instant, TimeZone timeZone) {
+    return SFTimestamp.fromNanoseconds(
+        instant.getEpochSecond() * Power10.intTable[9] + instant.getNano(), timeZone);
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
index 4462369c8..1f5b1458e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
@@ -205,14 +205,13 @@ private static long getInt64Value(
       case TIMESTAMP_NTZ:
         boolean ignoreTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
         longValue =
-            DataValidationUtil.validateAndParseTimestampNtzSb16(
-                    columnName, value, scale, ignoreTimezone)
+            DataValidationUtil.validateAndParseTimestamp(columnName, value, scale, ignoreTimezone)
                 .getTimeInScale()
                 .longValue();
         break;
       case TIMESTAMP_TZ:
         longValue =
-            DataValidationUtil.validateAndParseTimestampTz(columnName, value, scale)
+            DataValidationUtil.validateAndParseTimestamp(columnName, value, scale, false)
                 .getSfTimestamp()
                 .orElseThrow(
                     () ->
@@ -255,7 +254,7 @@ private static BigInteger getSb16Value(
       AbstractRowBuffer.ColumnPhysicalType physicalType) {
     switch (logicalType) {
       case TIMESTAMP_TZ:
-        return DataValidationUtil.validateAndParseTimestampTz(columnName, value, scale)
+        return DataValidationUtil.validateAndParseTimestamp(columnName, value, scale, false)
             .getSfTimestamp()
             .orElseThrow(
                 () ->
@@ -267,7 +266,7 @@ private static BigInteger getSb16Value(
       case TIMESTAMP_LTZ:
       case TIMESTAMP_NTZ:
         boolean ignoreTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
-        return DataValidationUtil.validateAndParseTimestampNtzSb16(
+        return DataValidationUtil.validateAndParseTimestamp(
                 columnName, value, scale, ignoreTimezone)
             .getTimeInScale();
       case FIXED:
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index d4caf2e16..bb5da4865 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -12,8 +12,7 @@
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseReal;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseString;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseTime;
-import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseTimestampNtzSb16;
-import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseTimestampTz;
+import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseTimestamp;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.validateAndParseVariant;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
@@ -40,6 +39,7 @@
 import java.util.Map;
 import java.util.Optional;
 import javax.xml.bind.DatatypeConverter;
+import net.snowflake.client.jdbc.internal.snowflake.common.core.SFTimestamp;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 import org.junit.Assert;
@@ -182,83 +182,73 @@ public void testValidateAndParseTime() {
   @Test
   public void testValidateAndParseTimestampNtzSb16() {
     assertEquals(
-        new TimestampWrapper(1609462800, 123000000, new BigInteger("1609462800123000000")),
-        DataValidationUtil.validateAndParseTimestampNtzSb16(
-            "COL", "2021-01-01 01:00:00.123", 9, true));
+        new TimestampWrapper(
+            1609462800,
+            123000000,
+            new BigInteger("1609462800123000000"),
+            SFTimestamp.fromNanoseconds(
+                BigDecimal.valueOf(1609462800 * 1_000_000_000L + 123000000))),
+        DataValidationUtil.validateAndParseTimestamp("COL", "2021-01-01 01:00:00.123", 9, true));
 
     // Time formats are not supported
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampNtzSb16("COL", "20:57:01.123456789+07:00", 3, false));
+        () -> validateAndParseTimestamp("COL", "20:57:01.123456789+07:00", 3, false));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampNtzSb16("COL", "20:57:01.123456789", 3, false));
+        () -> validateAndParseTimestamp("COL", "20:57:01.123456789", 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", "20:57:01", 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", "20:57", 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "20:57:01", 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "20:57", 3, false));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampNtzSb16("COL", "07:57:01.123456789 AM", 3, false));
+        () -> validateAndParseTimestamp("COL", "07:57:01.123456789 AM", 3, false));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampNtzSb16("COL", "04:01:07 AM", 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "04:01:07 AM", 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", "04:01 AM", 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "04:01 AM", 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", "04:01 PM", 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "04:01 PM", 3, false));
 
     // Test forbidden values
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampNtzSb16("COL", LocalTime.now(), 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampNtzSb16("COL", OffsetTime.now(), 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", new Date(), 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", LocalTime.now(), 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", 1.5f, 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", OffsetTime.now(), 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", 1.5, 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", new Date(), 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 1.5f, 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 1.5, 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "1.5", 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "1.0", 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", "1.5", 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", "1.0", 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampNtzSb16("COL", new Object(), 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", false, 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", "", 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", "foo", 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", new Object(), 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", false, 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "", 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "foo", 3, false));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampNtzSb16("COL", java.sql.Time.valueOf("20:57:00"), 3, false));
+        () -> validateAndParseTimestamp("COL", java.sql.Time.valueOf("20:57:00"), 3, false));
     expectError(
         ErrorCode.INVALID_ROW,
-        () ->
-            validateAndParseTimestampNtzSb16("COL", java.sql.Date.valueOf("2010-11-03"), 3, false));
+        () -> validateAndParseTimestamp("COL", java.sql.Date.valueOf("2010-11-03"), 3, false));
     expectError(
         ErrorCode.INVALID_ROW,
         () ->
-            validateAndParseTimestampNtzSb16(
+            validateAndParseTimestamp(
                 "COL", java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3, false));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampNtzSb16("COL", BigInteger.ZERO, 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampNtzSb16("COL", BigDecimal.ZERO, 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", BigInteger.ZERO, 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampNtzSb16("COL", 'c', 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", BigDecimal.ZERO, 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 'c', 3, false));
   }
 
   @Test
   public void testValidateAndPareTimestampTz() {
     TimestampWrapper result =
-        DataValidationUtil.validateAndParseTimestampTz("COL", "2021-01-01 01:00:00.123 +0100", 4);
+        DataValidationUtil.validateAndParseTimestamp(
+            "COL", "2021-01-01 01:00:00.123 +0100", 4, false);
     assertEquals(1609459200, result.getEpoch());
     assertEquals(123000000, result.getFraction());
     assertEquals(Optional.of(3600000), result.getTimezoneOffset());
@@ -267,48 +257,55 @@ public void testValidateAndPareTimestampTz() {
     // Time formats are not supported
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampTz("COL", "20:57:01.123456789+07:00", 3));
+        () -> validateAndParseTimestamp("COL", "20:57:01.123456789+07:00", 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "20:57:01.123456789", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "20:57:01", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "20:57", 3));
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestamp("COL", "20:57:01.123456789", 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "20:57:01", 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "20:57", 3, false));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampTz("COL", "07:57:01.123456789 AM", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "04:01:07 AM", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "04:01 AM", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "04:01 PM", 3));
+        () -> validateAndParseTimestamp("COL", "07:57:01.123456789 AM", 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "04:01:07 AM", 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "04:01 AM", 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "04:01 PM", 3, false));
 
     // Test forbidden values
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", LocalTime.now(), 3));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", LocalTime.now(), 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", OffsetTime.now(), 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", OffsetTime.now(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", new Date(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", 1.5f, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", 1.5, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "1.5", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "1.0", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", new Object(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", false, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", "foo", 3));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", new Date(), 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 1.5f, 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 1.5, 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "1.5", 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "1.0", 3, false));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", new Object(), 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", false, 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "", 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "foo", 3, false));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampTz("COL", java.sql.Time.valueOf("20:57:00"), 3));
+        () -> validateAndParseTimestamp("COL", java.sql.Time.valueOf("20:57:00"), 3, false));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestampTz("COL", java.sql.Date.valueOf("2010-11-03"), 3));
+        () -> validateAndParseTimestamp("COL", java.sql.Date.valueOf("2010-11-03"), 3, false));
     expectError(
         ErrorCode.INVALID_ROW,
         () ->
-            validateAndParseTimestampTz(
-                "COL", java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3));
+            validateAndParseTimestamp(
+                "COL", java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", BigInteger.ZERO, 3));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", BigInteger.ZERO, 3, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", BigDecimal.ZERO, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestampTz("COL", 'c', 3));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", BigDecimal.ZERO, 3, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 'c', 3, false));
   }
 
   @Test
@@ -961,14 +958,14 @@ public void testExceptionMessages() {
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
             + " be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
             + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
-        () -> validateAndParseTimestampNtzSb16("COL", new Object(), 3, true));
+        () -> validateAndParseTimestamp("COL", new Object(), 3, true));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
             + " Snowflake column COL of type TIMESTAMP: Not a valid timestamp, see"
             + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
             + " for the list of supported formats",
-        () -> validateAndParseTimestampNtzSb16("COL", "abc", 3, true));
+        () -> validateAndParseTimestamp("COL", "abc", 3, true));
 
     // TIMESTAMP_LTZ
     expectErrorCodeAndMessage(
@@ -976,14 +973,14 @@ public void testExceptionMessages() {
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
             + " be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
             + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
-        () -> validateAndParseTimestampNtzSb16("COL", new Object(), 3, false));
+        () -> validateAndParseTimestamp("COL", new Object(), 3, false));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
             + " Snowflake column COL of type TIMESTAMP: Not a valid timestamp, see"
             + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
             + " for the list of supported formats",
-        () -> validateAndParseTimestampNtzSb16("COL", "abc", 3, false));
+        () -> validateAndParseTimestamp("COL", "abc", 3, false));
 
     // TIMESTAMP_TZ
     expectErrorCodeAndMessage(
@@ -991,14 +988,14 @@ public void testExceptionMessages() {
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
             + " be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
             + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
-        () -> validateAndParseTimestampTz("COL", new Object(), 3));
+        () -> validateAndParseTimestamp("COL", new Object(), 3, false));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
             + " Snowflake column COL of type TIMESTAMP: Not a valid timestamp, see"
             + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
             + " for the list of supported formats",
-        () -> validateAndParseTimestampTz("COL", "abc", 3));
+        () -> validateAndParseTimestamp("COL", "abc", 3, false));
 
     // NUMBER
     expectErrorCodeAndMessage(
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
index 3a9445ea2..4abbf29db 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
@@ -862,12 +862,37 @@ public void testJavaTimeObjects() throws Exception {
         OffsetDateTime.parse("2007-12-03T10:15:30+01:00"),
         "2007-12-03",
         new StringProvider());
+    testIngestion(
+        "DATE",
+        OffsetDateTime.parse("2007-12-03T00:00:00+01:00"),
+        "2007-12-03",
+        new StringProvider());
+    testIngestion(
+        "DATE",
+        OffsetDateTime.parse("2007-12-03T00:00:00-08:00"),
+        "2007-12-03",
+        new StringProvider());
     testIngestion(
         "DATE",
         ZonedDateTime.parse("2007-12-03T10:15:30+01:00[Europe/Paris]"),
         "2007-12-03",
         new StringProvider());
-
+    testIngestion(
+        "DATE",
+        ZonedDateTime.parse("2007-12-03T00:00:00+01:00[Europe/Paris]"),
+        "2007-12-03",
+        new StringProvider());
+    testIngestion(
+        "DATE",
+        ZonedDateTime.parse("2007-12-03T00:00:00-08:00[America/Los_Angeles]"),
+        "2007-12-03",
+        new StringProvider());
+    testIngestion(
+        "DATE",
+        ZonedDateTime.parse("2007-07-03T00:00:00-07:00[America/Los_Angeles]"),
+        "2007-07-03",
+        new StringProvider());
+    //
     // TIMESTAMP_NTZ (LocalDate, LocalDateTime, OffsetDateTime, ZonedDateTime are supported)
     testIngestion(
         "TIMESTAMP_NTZ",
@@ -894,6 +919,11 @@ public void testJavaTimeObjects() throws Exception {
         ZonedDateTime.parse("2007-12-03T10:15:30.123456789+01:00[Europe/Paris]"),
         "2007-12-03 10:15:30.123456789 Z",
         new StringProvider());
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        ZonedDateTime.parse("2007-07-03T10:15:30.123456789+02:00[Europe/Paris]"),
+        "2007-07-03 10:15:30.123456789 Z",
+        new StringProvider());
 
     useLosAngelesTimeZone();
     // TIMESTAMP_LTZ (LocalDate, LocalDateTime, OffsetDateTime, ZonedDateTime are supported)
@@ -922,6 +952,11 @@ public void testJavaTimeObjects() throws Exception {
         ZonedDateTime.parse("2007-12-03T10:15:30.123456789+01:00[Europe/Paris]"),
         "2007-12-03 01:15:30.123456789 -0800",
         new StringProvider());
+    testIngestion(
+        "TIMESTAMP_LTZ",
+        ZonedDateTime.parse("2007-07-03T10:15:30.123456789+02:00[Europe/Paris]"),
+        "2007-07-03 01:15:30.123456789 -0700",
+        new StringProvider());
 
     // TIMESTAMP_TZ (LocalDate, LocalDateTime, OffsetDateTime, ZonedDateTime are supported)
     testIngestion(
@@ -949,6 +984,11 @@ public void testJavaTimeObjects() throws Exception {
         ZonedDateTime.parse("2007-12-03T10:15:30.123456789+01:00[Europe/Paris]"),
         "2007-12-03 10:15:30.123456789 +0100",
         new StringProvider());
+    testIngestion(
+        "TIMESTAMP_TZ",
+        ZonedDateTime.parse("2007-07-03T10:15:30.123456789+02:00[Europe/Paris]"),
+        "2007-07-03 10:15:30.123456789 +0200",
+        new StringProvider());
   }
 
   @Test

From 4d9957a41a02c97c334198dbaf2e30d34815887a Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 12 Jan 2023 11:57:51 -0800
Subject: [PATCH 111/356] SNOW-701710: [Client side] Add telemetry to track the
 end2end latency for every BDEC (#295)

In order to track the exact end2end latency for Snowpipe Streaming, we're adding two more fields in the BDEC registration request (first_insert_time_in_ms and last_insert_time_in_ms) which corresponds to the first row insert timestamp in the BDEC and the last row insert timestamp in the BDEC, then we could use it to calculate the end2end latency for each BDEC at server side and output it in the telemetry
---
 .../streaming/internal/AbstractRowBuffer.java | 11 ++++++--
 .../streaming/internal/ArrowFlusher.java      |  8 +++++-
 .../streaming/internal/BlobBuilder.java       | 12 ++++----
 .../streaming/internal/ChannelData.java       | 22 +++++++++++++++
 .../internal/ChannelRuntimeState.java         | 26 +++++++++++++++--
 .../streaming/internal/ChunkMetadata.java     | 28 +++++++++++++++++++
 .../streaming/internal/FlushService.java      |  3 ++
 .../ingest/streaming/internal/Flusher.java    |  6 +++-
 .../streaming/internal/ParquetFlusher.java    |  9 +++++-
 ...owflakeStreamingIngestChannelInternal.java |  2 +-
 .../streaming/internal/FlushServiceTest.java  |  4 +++
 .../SnowflakeStreamingIngestChannelTest.java  |  4 +++
 .../SnowflakeStreamingIngestClientTest.java   |  8 ++++++
 13 files changed, 130 insertions(+), 13 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 4f993534e..a9942e502 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -20,6 +20,7 @@
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
 import net.snowflake.ingest.utils.SFException;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.util.VisibleForTesting;
@@ -152,10 +153,10 @@ public int getOrdinal() {
   // Names of non-nullable columns
   private final Set<String> nonNullableFieldNames;
 
-  // buffer's channel fully qualified name with database, schema and table
+  // Buffer's channel fully qualified name with database, schema and table
   final String channelFullyQualifiedName;
 
-  // metric callback to report size of inserted rows
+  // Metric callback to report size of inserted rows
   private final Consumer<Float> rowSizeMetric;
 
   // Allocator used to allocate the buffers
@@ -279,6 +280,7 @@ public InsertValidationResponse insertRows(
     InsertValidationResponse response = new InsertValidationResponse();
     this.flushLock.lock();
     try {
+      this.channelState.updateInsertStats(System.currentTimeMillis(), this.rowCount);
       if (onErrorOption == OpenChannelRequest.OnErrorOption.CONTINUE) {
         // Used to map incoming row(nth row) to InsertError(for nth row) in response
         long rowIndex = 0;
@@ -356,6 +358,7 @@ public ChannelData<T> flush(final String filePath) {
       long oldRowSequencer = 0;
       String oldOffsetToken = null;
       Map<String, RowBufferStats> oldColumnEps = null;
+      Pair<Long, Long> oldMinMaxInsertTimeInMs = null;
 
       logger.logDebug(
           "Arrow buffer flush about to take lock on channel={}", channelFullyQualifiedName);
@@ -370,6 +373,9 @@ public ChannelData<T> flush(final String filePath) {
           oldRowSequencer = this.channelState.incrementAndGetRowSequencer();
           oldOffsetToken = this.channelState.getOffsetToken();
           oldColumnEps = new HashMap<>(this.statsMap);
+          oldMinMaxInsertTimeInMs =
+              new Pair<>(
+                  this.channelState.getFirstInsertInMs(), this.channelState.getLastInsertInMs());
           // Reset everything in the buffer once we save all the info
           reset();
         }
@@ -391,6 +397,7 @@ public ChannelData<T> flush(final String filePath) {
         data.setRowSequencer(oldRowSequencer);
         data.setOffsetToken(oldOffsetToken);
         data.setColumnEps(oldColumnEps);
+        data.setMinMaxInsertTimeInMs(oldMinMaxInsertTimeInMs);
         data.setFlusherFactory(this::createFlusher);
         return data;
       }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
index 3eca44154..7520b39d8 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
@@ -11,6 +11,7 @@
 import java.util.Map;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
 import net.snowflake.ingest.utils.SFException;
 import org.apache.arrow.vector.VectorLoader;
 import org.apache.arrow.vector.VectorSchemaRoot;
@@ -39,6 +40,7 @@ public Flusher.SerializationResult serialize(
     VectorLoader loader = null;
     String firstChannelFullyQualifiedTableName = null;
     Map<String, RowBufferStats> columnEpStatsMapCombined = null;
+    Pair<Long, Long> chunkMinMaxInsertTimeInMs = null;
 
     try {
       for (ChannelData<VectorSchemaRoot> data : channelsDataPerTable) {
@@ -67,6 +69,7 @@ public Flusher.SerializationResult serialize(
           firstChannelFullyQualifiedTableName =
               data.getChannelContext().getFullyQualifiedTableName();
           arrowWriter.start();
+          chunkMinMaxInsertTimeInMs = data.getMinMaxInsertTimeInMs();
         } else {
           // This method assumes that channelsDataPerTable is grouped by table. We double check
           // here and throw an error if the assumption is violated
@@ -78,6 +81,9 @@ public Flusher.SerializationResult serialize(
 
           columnEpStatsMapCombined =
               ChannelData.getCombinedColumnStatsMap(columnEpStatsMapCombined, data.getColumnEps());
+          chunkMinMaxInsertTimeInMs =
+              ChannelData.getCombinedMinMaxInsertTimeInMs(
+                  chunkMinMaxInsertTimeInMs, data.getMinMaxInsertTimeInMs());
 
           VectorUnloader unloader = new VectorUnloader(data.getVectors());
           ArrowRecordBatch recordBatch = unloader.getRecordBatch();
@@ -104,6 +110,6 @@ public Flusher.SerializationResult serialize(
       }
     }
     return new Flusher.SerializationResult(
-        channelsMetadataList, columnEpStatsMapCombined, rowCount);
+        channelsMetadataList, columnEpStatsMapCombined, rowCount, chunkMinMaxInsertTimeInMs);
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
index cac9d185a..9b4cf70a9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
@@ -82,10 +82,10 @@ static <T> Blob constructBlobAndMetadata(
           channelsDataPerTable.get(0).getChannelContext();
 
       Flusher<T> flusher = channelsDataPerTable.get(0).createFlusher();
-      Flusher.SerializationResult result =
+      Flusher.SerializationResult serializedChunk =
           flusher.serialize(channelsDataPerTable, chunkData, filePath);
 
-      if (!result.channelsMetadataList.isEmpty()) {
+      if (!serializedChunk.channelsMetadataList.isEmpty()) {
         Pair<byte[], Integer> compressionResult =
             compressIfNeededAndPadChunk(
                 filePath,
@@ -123,12 +123,14 @@ static <T> Blob constructBlobAndMetadata(
                 // The compressedChunkLength is used because it is the actual data size used for
                 // decompression and md5 calculation on server side.
                 .setChunkLength(compressedChunkLength)
-                .setChannelList(result.channelsMetadataList)
+                .setChannelList(serializedChunk.channelsMetadataList)
                 .setChunkMD5(md5)
                 .setEncryptionKeyId(firstChannelFlushContext.getEncryptionKeyId())
                 .setEpInfo(
                     AbstractRowBuffer.buildEpInfoFromStats(
-                        result.rowCount, result.columnEpStatsMapCombined))
+                        serializedChunk.rowCount, serializedChunk.columnEpStatsMapCombined))
+                .setFirstInsertTimeInMs(serializedChunk.chunkMinMaxInsertTimeInMs.getFirst())
+                .setLastInsertTimeInMs(serializedChunk.chunkMinMaxInsertTimeInMs.getSecond())
                 .build();
 
         // Add chunk metadata and data to the list
@@ -143,7 +145,7 @@ static <T> Blob constructBlobAndMetadata(
                 + " bdecVersion={}",
             filePath,
             firstChannelFlushContext.getFullyQualifiedTableName(),
-            result.rowCount,
+            serializedChunk.rowCount,
             startOffset,
             chunkData.size(),
             compressedChunkLength,
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
index 234d72ca9..ffc327d9d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
@@ -8,6 +8,7 @@
 import java.util.Map;
 import java.util.function.Supplier;
 import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Pair;
 import net.snowflake.ingest.utils.SFException;
 
 /**
@@ -23,6 +24,7 @@ class ChannelData<T> {
   private float bufferSize;
   private int rowCount;
   private Map<String, RowBufferStats> columnEps;
+  private Pair<Long, Long> minMaxInsertTimeInMs;
   private ChannelFlushContext channelFlushContext;
   private Supplier<Flusher<T>> flusherFactory;
 
@@ -58,6 +60,18 @@ public static Map<String, RowBufferStats> getCombinedColumnStatsMap(
     return result;
   }
 
+  /**
+   * Combines the two paris of min/max insert timestamp together
+   *
+   * @return A new pair which the first element is min(left min, right min) and the second element
+   *     is max(left max, right max)
+   */
+  public static Pair<Long, Long> getCombinedMinMaxInsertTimeInMs(
+      Pair<Long, Long> left, Pair<Long, Long> right) {
+    return new Pair<>(
+        Math.min(left.getFirst(), right.getFirst()), Math.max(left.getSecond(), right.getSecond()));
+  }
+
   public Map<String, RowBufferStats> getColumnEps() {
     return columnEps;
   }
@@ -122,6 +136,14 @@ public void setFlusherFactory(Supplier<Flusher<T>> flusherFactory) {
     this.flusherFactory = flusherFactory;
   }
 
+  Pair<Long, Long> getMinMaxInsertTimeInMs() {
+    return this.minMaxInsertTimeInMs;
+  }
+
+  void setMinMaxInsertTimeInMs(Pair<Long, Long> minMaxInsertTimeInMs) {
+    this.minMaxInsertTimeInMs = minMaxInsertTimeInMs;
+  }
+
   @Override
   public String toString() {
     return "ChannelData{"
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelRuntimeState.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelRuntimeState.java
index 3d3e5ac1f..26314db3e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelRuntimeState.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelRuntimeState.java
@@ -11,12 +11,16 @@ class ChannelRuntimeState {
   // Indicates whether the channel is still valid
   private volatile boolean isValid;
 
-  // the channel's current offset token
+  // The channel's current offset token
   private volatile String offsetToken;
 
-  // the channel's current row sequencer
+  // The channel's current row sequencer
   private final AtomicLong rowSequencer;
 
+  // First and last insert time in ms, used for end2end latency measurement
+  private Long firstInsertInMs;
+  private Long lastInsertInMs;
+
   ChannelRuntimeState(String offsetToken, long rowSequencer, boolean isValid) {
     this.offsetToken = offsetToken;
     this.rowSequencer = new AtomicLong(rowSequencer);
@@ -60,4 +64,22 @@ long getRowSequencer() {
   void setOffsetToken(String offsetToken) {
     this.offsetToken = offsetToken;
   }
+
+  /** Update the insert stats for the current row buffer whenever needed */
+  void updateInsertStats(long currentTimeInMs, int rowCount) {
+    if (rowCount == 0) {
+      this.firstInsertInMs = currentTimeInMs;
+    }
+    this.lastInsertInMs = currentTimeInMs;
+  }
+
+  /** Get the insert timestamp of the first row in the current row buffer */
+  Long getFirstInsertInMs() {
+    return this.firstInsertInMs;
+  }
+
+  /** Get the insert timestamp of the last row in the current row buffer */
+  Long getLastInsertInMs() {
+    return this.lastInsertInMs;
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChunkMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChunkMetadata.java
index 2257026cd..582c7e272 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChunkMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChunkMetadata.java
@@ -19,6 +19,8 @@ class ChunkMetadata {
   private final String chunkMD5;
   private final EpInfo epInfo;
   private final Long encryptionKeyId;
+  private final Long firstInsertTimeInMs;
+  private final Long lastInsertTimeInMs;
 
   static Builder builder() {
     return new Builder();
@@ -35,6 +37,8 @@ static class Builder {
     private String chunkMD5;
     private EpInfo epInfo;
     private Long encryptionKeyId;
+    private Long firstInsertTimeInMs;
+    private Long lastInsertTimeInMs;
 
     Builder setOwningTableFromChannelContext(ChannelFlushContext channelFlushContext) {
       this.dbName = channelFlushContext.getDbName();
@@ -73,6 +77,16 @@ Builder setEncryptionKeyId(Long encryptionKeyId) {
       return this;
     }
 
+    Builder setFirstInsertTimeInMs(Long firstInsertTimeInMs) {
+      this.firstInsertTimeInMs = firstInsertTimeInMs;
+      return this;
+    }
+
+    Builder setLastInsertTimeInMs(Long lastInsertTimeInMs) {
+      this.lastInsertTimeInMs = lastInsertTimeInMs;
+      return this;
+    }
+
     ChunkMetadata build() {
       return new ChunkMetadata(this);
     }
@@ -88,6 +102,8 @@ private ChunkMetadata(Builder builder) {
     Utils.assertNotNull("chunk MD5", builder.chunkMD5);
     Utils.assertNotNull("chunk ep info", builder.epInfo);
     Utils.assertNotNull("encryption key id", builder.encryptionKeyId);
+    Utils.assertNotNull("chunk first insert time in ms", builder.firstInsertTimeInMs);
+    Utils.assertNotNull("chunk last insert time in ms", builder.lastInsertTimeInMs);
 
     this.dbName = builder.dbName;
     this.schemaName = builder.schemaName;
@@ -98,6 +114,8 @@ private ChunkMetadata(Builder builder) {
     this.chunkMD5 = builder.chunkMD5;
     this.epInfo = builder.epInfo;
     this.encryptionKeyId = builder.encryptionKeyId;
+    this.firstInsertTimeInMs = builder.firstInsertTimeInMs;
+    this.lastInsertTimeInMs = builder.lastInsertTimeInMs;
   }
 
   /**
@@ -153,4 +171,14 @@ EpInfo getEpInfo() {
   Long getEncryptionKeyId() {
     return this.encryptionKeyId;
   }
+
+  @JsonProperty("first_insert_time_in_ms")
+  Long getFirstInsertTimeInMs() {
+    return this.firstInsertTimeInMs;
+  }
+
+  @JsonProperty("last_insert_time_in_ms")
+  Long getLastInsertTimeInMs() {
+    return this.lastInsertTimeInMs;
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 93245eb9d..cfa0417f7 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -440,7 +440,10 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
           NoSuchPaddingException, IllegalBlockSizeException, BadPaddingException,
           InvalidKeyException {
     Timer.Context buildContext = Utils.createTimerContext(this.owningClient.buildLatency);
+
+    // Construct the blob along with the metadata of the blob
     BlobBuilder.Blob blob = BlobBuilder.constructBlobAndMetadata(filePath, blobData, bdecVersion);
+
     if (buildContext != null) {
       buildContext.stop();
     }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
index 37ab4afc4..762886bd0 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
@@ -8,6 +8,7 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import net.snowflake.ingest.utils.Pair;
 
 /**
  * Interface to convert {@link ChannelData} buffered in {@link RowBuffer} to the underlying format
@@ -35,14 +36,17 @@ class SerializationResult {
     final List<ChannelMetadata> channelsMetadataList;
     final Map<String, RowBufferStats> columnEpStatsMapCombined;
     final long rowCount;
+    final Pair<Long, Long> chunkMinMaxInsertTimeInMs;
 
     public SerializationResult(
         List<ChannelMetadata> channelsMetadataList,
         Map<String, RowBufferStats> columnEpStatsMapCombined,
-        long rowCount) {
+        long rowCount,
+        Pair<Long, Long> chunkMinMaxInsertTimeInMs) {
       this.channelsMetadataList = channelsMetadataList;
       this.columnEpStatsMapCombined = columnEpStatsMapCombined;
       this.rowCount = rowCount;
+      this.chunkMinMaxInsertTimeInMs = chunkMinMaxInsertTimeInMs;
     }
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
index e5b496ff5..5772c6d19 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
@@ -15,6 +15,7 @@
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
 import net.snowflake.ingest.utils.SFException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.parquet.column.ColumnDescriptor;
@@ -56,6 +57,7 @@ public SerializationResult serialize(
     String firstChannelFullyQualifiedTableName = null;
     Map<String, RowBufferStats> columnEpStatsMapCombined = null;
     List<List<Object>> rows = null;
+    Pair<Long, Long> chunkMinMaxInsertTimeInMs = null;
 
     for (ChannelData<ParquetChunkData> data : channelsDataPerTable) {
       // Create channel metadata
@@ -79,6 +81,7 @@ public SerializationResult serialize(
         columnEpStatsMapCombined = data.getColumnEps();
         rows = new ArrayList<>();
         firstChannelFullyQualifiedTableName = data.getChannelContext().getFullyQualifiedTableName();
+        chunkMinMaxInsertTimeInMs = data.getMinMaxInsertTimeInMs();
       } else {
         // This method assumes that channelsDataPerTable is grouped by table. We double check
         // here and throw an error if the assumption is violated
@@ -90,6 +93,9 @@ public SerializationResult serialize(
 
         columnEpStatsMapCombined =
             ChannelData.getCombinedColumnStatsMap(columnEpStatsMapCombined, data.getColumnEps());
+        chunkMinMaxInsertTimeInMs =
+            ChannelData.getCombinedMinMaxInsertTimeInMs(
+                chunkMinMaxInsertTimeInMs, data.getMinMaxInsertTimeInMs());
       }
       rows.addAll(data.getVectors().rows);
 
@@ -106,7 +112,8 @@ public SerializationResult serialize(
     Map<String, String> metadata = channelsDataPerTable.get(0).getVectors().metadata;
 
     flushToParquetBdecChunk(chunkData, rows, metadata, channelsMetadataList);
-    return new SerializationResult(channelsMetadataList, columnEpStatsMapCombined, rowCount);
+    return new SerializationResult(
+        channelsMetadataList, columnEpStatsMapCombined, rowCount, chunkMinMaxInsertTimeInMs);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 124e1cac6..8eae060cd 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -49,7 +49,7 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
   // Reference to the client that owns this channel
   private final SnowflakeStreamingIngestClientInternal<T> owningClient;
 
-  // state of the channel that will be shared with its underlying buffer
+  // State of the channel that will be shared with its underlying buffer
   private final ChannelRuntimeState channelState;
 
   /**
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 772e7057e..884b7fde0 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -530,6 +530,8 @@ public void testBuildAndUpload() throws Exception {
             .setChunkMD5("md5")
             .setEncryptionKeyId(1234L)
             .setEpInfo(expectedChunkEpInfo)
+            .setFirstInsertTimeInMs(1L)
+            .setLastInsertTimeInMs(2L)
             .build();
 
     // Check FlushService.upload called with correct arguments
@@ -719,6 +721,8 @@ public void testBlobBuilder() throws Exception {
             .setChunkMD5("md5")
             .setEncryptionKeyId(1234L)
             .setEpInfo(epInfo)
+            .setFirstInsertTimeInMs(1L)
+            .setLastInsertTimeInMs(2L)
             .build();
 
     chunksMetadataList.add(chunkMetadata);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 162304aae..8b731112a 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -497,10 +497,12 @@ public void testInsertRow() {
     ChannelData<VectorSchemaRoot> data = channel.getData("my_snowpipe_streaming.bdec");
     Assert.assertNull(data);
 
+    long insertStartTimeInMs = System.currentTimeMillis();
     InsertValidationResponse response = channel.insertRow(row, "1");
     Assert.assertFalse(response.hasErrors());
     response = channel.insertRows(Collections.singletonList(row), "2");
     Assert.assertFalse(response.hasErrors());
+    long insertEndTimeInMs = System.currentTimeMillis();
 
     // Get data again to verify the row is inserted
     data = channel.getData("my_snowpipe_streaming.bdec");
@@ -509,6 +511,8 @@ public void testInsertRow() {
     Assert.assertEquals(1, data.getVectors().getFieldVectors().size());
     Assert.assertEquals("2", data.getOffsetToken());
     Assert.assertTrue(data.getBufferSize() > 0);
+    Assert.assertTrue(insertStartTimeInMs <= data.getMinMaxInsertTimeInMs().getFirst());
+    Assert.assertTrue(insertEndTimeInMs >= data.getMinMaxInsertTimeInMs().getSecond());
   }
 
   @Test
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index 91c40eb13..7d5e52818 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -472,6 +472,8 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
             .setChunkMD5("md5")
             .setEncryptionKeyId(1234L)
             .setEpInfo(epInfo)
+            .setFirstInsertTimeInMs(1L)
+            .setLastInsertTimeInMs(2L)
             .build();
 
     List<BlobMetadata> blobs =
@@ -545,6 +547,8 @@ private Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> getRetryBlobMetadata(
             .setChunkMD5("md51")
             .setEncryptionKeyId(1234L)
             .setEpInfo(epInfo)
+            .setFirstInsertTimeInMs(1L)
+            .setLastInsertTimeInMs(2L)
             .build();
     ChunkMetadata chunkMetadata2 =
         ChunkMetadata.builder()
@@ -555,6 +559,8 @@ private Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> getRetryBlobMetadata(
             .setChunkMD5("md52")
             .setEncryptionKeyId(1234L)
             .setEpInfo(epInfo)
+            .setFirstInsertTimeInMs(1L)
+            .setLastInsertTimeInMs(2L)
             .build();
     ChunkMetadata chunkMetadata3 =
         ChunkMetadata.builder()
@@ -565,6 +571,8 @@ private Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> getRetryBlobMetadata(
             .setChunkMD5("md53")
             .setEncryptionKeyId(1234L)
             .setEpInfo(epInfo)
+            .setFirstInsertTimeInMs(1L)
+            .setLastInsertTimeInMs(2L)
             .build();
 
     chunks1.add(chunkMetadata1);

From 8ea3e165b080cbb9b7afba329084ed1329dd2934 Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Fri, 13 Jan 2023 23:48:54 +0100
Subject: [PATCH 112/356] SNOW-690793 add enable_parquet (#299)

---
 .../streaming/internal/AbstractRowBuffer.java |  44 ++-
 .../streaming/internal/ArrowFlusher.java      |  11 +-
 .../streaming/internal/ArrowRowBuffer.java    |  27 +-
 .../streaming/internal/BlobBuilder.java       |  11 +-
 .../streaming/internal/FlushService.java      |   8 +-
 .../ingest/streaming/internal/Flusher.java    |   7 +-
 .../streaming/internal/ParquetChunkData.java  |  18 +-
 .../streaming/internal/ParquetFlusher.java    | 359 +++++-------------
 .../streaming/internal/ParquetRowBuffer.java  |  91 ++++-
 ...owflakeStreamingIngestChannelInternal.java |   7 +-
 .../ingest/utils/ParameterProvider.java       |  20 +
 .../parquet/hadoop/BdecParquetReader.java     | 239 ++++++++++++
 .../parquet/hadoop/BdecParquetWriter.java     | 291 ++++++++++++++
 .../streaming/internal/RowBufferTest.java     |  13 +-
 14 files changed, 826 insertions(+), 320 deletions(-)
 create mode 100644 src/main/java/org/apache/parquet/hadoop/BdecParquetReader.java
 create mode 100644 src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index a9942e502..bfd68b871 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -22,6 +22,7 @@
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.Pair;
 import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.Utils;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.util.VisibleForTesting;
 
@@ -472,6 +473,41 @@ void reset() {
   @VisibleForTesting
   abstract int getTempRowCount();
 
+  /**
+   * Close row buffer by releasing its internal resources only. Note that the allocated memory for
+   * the channel is released elsewhere (in close).
+   */
+  abstract void closeInternal();
+
+  /** Close the row buffer and release allocated memory for the channel. */
+  @Override
+  public synchronized void close(String name) {
+    long allocatedBeforeRelease = this.allocator.getAllocatedMemory();
+
+    closeInternal();
+
+    long allocatedAfterRelease = this.allocator.getAllocatedMemory();
+    logger.logInfo(
+        "Trying to close {} for channel={} from function={}, allocatedBeforeRelease={},"
+            + " allocatedAfterRelease={}",
+        this.getClass().getSimpleName(),
+        channelFullyQualifiedName,
+        name,
+        allocatedBeforeRelease,
+        allocatedAfterRelease);
+    Utils.closeAllocator(this.allocator);
+
+    // If the channel is valid but still has leftover data, throw an exception because it should be
+    // cleaned up already before calling close
+    if (allocatedBeforeRelease > 0 && this.channelState.isValid()) {
+      throw new SFException(
+          ErrorCode.INTERNAL_ERROR,
+          String.format(
+              "Memory leaked=%d by allocator=%s, channel=%s",
+              allocatedBeforeRelease, this.allocator, channelFullyQualifiedName));
+    }
+  }
+
   /**
    * Given a set of col names to stats, build the right ep info
    *
@@ -497,7 +533,9 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
       Constants.BdecVersion bdecVersion,
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
-      ChannelRuntimeState channelRuntimeState) {
+      ChannelRuntimeState channelRuntimeState,
+      boolean bufferForTests,
+      boolean enableParquetMemoryOptimization) {
     switch (bdecVersion) {
       case ONE:
         //noinspection unchecked
@@ -516,7 +554,9 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
                 allocator,
                 fullyQualifiedChannelName,
                 rowSizeMetric,
-                channelRuntimeState);
+                channelRuntimeState,
+                bufferForTests,
+                enableParquetMemoryOptimization);
       default:
         throw new SFException(
             ErrorCode.INTERNAL_ERROR, "Unsupported BDEC format version: " + bdecVersion);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
index 7520b39d8..42452cb55 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
@@ -29,10 +29,9 @@ public class ArrowFlusher implements Flusher<VectorSchemaRoot> {
 
   @Override
   public Flusher.SerializationResult serialize(
-      List<ChannelData<VectorSchemaRoot>> channelsDataPerTable,
-      ByteArrayOutputStream chunkData,
-      String filePath)
+      List<ChannelData<VectorSchemaRoot>> channelsDataPerTable, String filePath)
       throws IOException {
+    ByteArrayOutputStream chunkData = new ByteArrayOutputStream();
     List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
     long rowCount = 0L;
     VectorSchemaRoot root = null;
@@ -110,6 +109,10 @@ public Flusher.SerializationResult serialize(
       }
     }
     return new Flusher.SerializationResult(
-        channelsMetadataList, columnEpStatsMapCombined, rowCount, chunkMinMaxInsertTimeInMs);
+        channelsMetadataList,
+        columnEpStatsMapCombined,
+        rowCount,
+        chunkData,
+        chunkMinMaxInsertTimeInMs);
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index af385ed26..34b924177 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -122,37 +122,14 @@ public void setupSchema(List<ColumnMetadata> columns) {
     this.tempVectorsRoot = new VectorSchemaRoot(tempVectors);
   }
 
-  /**
-   * Close the row buffer and release resources. Note that the caller needs to handle
-   * synchronization
-   */
+  /** Close the row buffer by releasing its internal resources. */
   @Override
-  public void close(String name) {
-    long allocatedBeforeRelease = this.allocator.getAllocatedMemory();
+  public void closeInternal() {
     if (this.vectorsRoot != null) {
       this.vectorsRoot.close();
       this.tempVectorsRoot.close();
     }
     this.fields.clear();
-    long allocatedAfterRelease = this.allocator.getAllocatedMemory();
-    logger.logInfo(
-        "Trying to close arrow buffer for channel={} from function={}, allocatedBeforeRelease={},"
-            + " allocatedAfterRelease={}",
-        channelFullyQualifiedName,
-        name,
-        allocatedBeforeRelease,
-        allocatedAfterRelease);
-    Utils.closeAllocator(this.allocator);
-
-    // If the channel is valid but still has leftover data, throw an exception because it should be
-    // cleaned up already before calling close
-    if (allocatedBeforeRelease > 0 && channelState.isValid()) {
-      throw new SFException(
-          ErrorCode.INTERNAL_ERROR,
-          String.format(
-              "Memory leaked=%d by allocator=%s, channel=%s",
-              allocatedBeforeRelease, this.allocator, channelFullyQualifiedName));
-    }
   }
 
   /** Reset the variables after each flush. Note that the caller needs to handle synchronization */
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
index 9b4cf70a9..2ae31c42d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
@@ -63,10 +63,15 @@ class BlobBuilder {
    * @param blobData All the data for one blob. Assumes that all ChannelData in the inner List
    *     belongs to the same table. Will error if this is not the case
    * @param bdecVersion version of blob
+   * @param enableParquetMemoryOptimization indicates whether Parquet memory optimization should be
+   *     applied
    * @return {@link Blob} data
    */
   static <T> Blob constructBlobAndMetadata(
-      String filePath, List<List<ChannelData<T>>> blobData, Constants.BdecVersion bdecVersion)
+      String filePath,
+      List<List<ChannelData<T>>> blobData,
+      Constants.BdecVersion bdecVersion,
+      boolean enableParquetMemoryOptimization)
       throws IOException, NoSuchPaddingException, NoSuchAlgorithmException,
           InvalidAlgorithmParameterException, InvalidKeyException, IllegalBlockSizeException,
           BadPaddingException {
@@ -77,15 +82,15 @@ static <T> Blob constructBlobAndMetadata(
 
     // TODO: channels with different schema can't be combined even if they belongs to same table
     for (List<ChannelData<T>> channelsDataPerTable : blobData) {
-      ByteArrayOutputStream chunkData = new ByteArrayOutputStream();
       ChannelFlushContext firstChannelFlushContext =
           channelsDataPerTable.get(0).getChannelContext();
 
       Flusher<T> flusher = channelsDataPerTable.get(0).createFlusher();
       Flusher.SerializationResult serializedChunk =
-          flusher.serialize(channelsDataPerTable, chunkData, filePath);
+          flusher.serialize(channelsDataPerTable, filePath);
 
       if (!serializedChunk.channelsMetadataList.isEmpty()) {
+        ByteArrayOutputStream chunkData = serializedChunk.chunkData;
         Pair<byte[], Integer> compressionResult =
             compressIfNeededAndPadChunk(
                 filePath,
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index cfa0417f7..e1c3d01e9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -442,8 +442,12 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
     Timer.Context buildContext = Utils.createTimerContext(this.owningClient.buildLatency);
 
     // Construct the blob along with the metadata of the blob
-    BlobBuilder.Blob blob = BlobBuilder.constructBlobAndMetadata(filePath, blobData, bdecVersion);
-
+    BlobBuilder.Blob blob =
+        BlobBuilder.constructBlobAndMetadata(
+            filePath,
+            blobData,
+            bdecVersion,
+            owningClient.getParameterProvider().getEnableParquetInternalBuffering());
     if (buildContext != null) {
       buildContext.stop();
     }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
index 762886bd0..9923d2485 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
@@ -22,13 +22,11 @@ public interface Flusher<T> {
    * Serialize buffered rows into the underlying format.
    *
    * @param channelsDataPerTable buffered rows
-   * @param chunkData output
    * @param filePath file path
    * @return {@link SerializationResult}
    * @throws IOException
    */
-  SerializationResult serialize(
-      List<ChannelData<T>> channelsDataPerTable, ByteArrayOutputStream chunkData, String filePath)
+  SerializationResult serialize(List<ChannelData<T>> channelsDataPerTable, String filePath)
       throws IOException;
 
   /** Holds result of the buffered rows conversion: channel metadata and stats. */
@@ -36,16 +34,19 @@ class SerializationResult {
     final List<ChannelMetadata> channelsMetadataList;
     final Map<String, RowBufferStats> columnEpStatsMapCombined;
     final long rowCount;
+    final ByteArrayOutputStream chunkData;
     final Pair<Long, Long> chunkMinMaxInsertTimeInMs;
 
     public SerializationResult(
         List<ChannelMetadata> channelsMetadataList,
         Map<String, RowBufferStats> columnEpStatsMapCombined,
         long rowCount,
+        ByteArrayOutputStream chunkData,
         Pair<Long, Long> chunkMinMaxInsertTimeInMs) {
       this.channelsMetadataList = channelsMetadataList;
       this.columnEpStatsMapCombined = columnEpStatsMapCombined;
       this.rowCount = rowCount;
+      this.chunkData = chunkData;
       this.chunkMinMaxInsertTimeInMs = chunkMinMaxInsertTimeInMs;
     }
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
index 0c7b12ff2..16b1ededa 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetChunkData.java
@@ -4,22 +4,36 @@
 
 package net.snowflake.ingest.streaming.internal;
 
+import java.io.ByteArrayOutputStream;
 import java.util.List;
 import java.util.Map;
+import org.apache.parquet.hadoop.BdecParquetWriter;
 
 /** Parquet data holder to buffer rows. */
 public class ParquetChunkData {
+  // buffered rows serialized into Java objects. Needed for the Parquet w/o memory optimization.
   final List<List<Object>> rows;
+
+  final BdecParquetWriter parquetWriter;
+  final ByteArrayOutputStream output;
   final Map<String, String> metadata;
 
   /**
    * Construct parquet data chunk.
    *
-   * @param rows chunk row set
+   * @param rows buffered row data as a list
+   * @param parquetWriter buffered parquet row data
+   * @param output byte array file output
    * @param metadata chunk metadata
    */
-  public ParquetChunkData(List<List<Object>> rows, Map<String, String> metadata) {
+  public ParquetChunkData(
+      List<List<Object>> rows,
+      BdecParquetWriter parquetWriter,
+      ByteArrayOutputStream output,
+      Map<String, String> metadata) {
     this.rows = rows;
+    this.parquetWriter = parquetWriter;
+    this.output = output;
     this.metadata = metadata;
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
index 5772c6d19..9382a673c 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
@@ -4,34 +4,18 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.MAX_CHUNK_SIZE_IN_BYTES;
-
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
-import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.Pair;
 import net.snowflake.ingest.utils.SFException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.parquet.column.ColumnDescriptor;
-import org.apache.parquet.column.ParquetProperties;
-import org.apache.parquet.hadoop.ParquetFileWriter;
-import org.apache.parquet.hadoop.ParquetWriter;
-import org.apache.parquet.hadoop.api.WriteSupport;
-import org.apache.parquet.hadoop.metadata.CompressionCodecName;
-import org.apache.parquet.io.DelegatingPositionOutputStream;
-import org.apache.parquet.io.OutputFile;
-import org.apache.parquet.io.ParquetEncodingException;
-import org.apache.parquet.io.PositionOutputStream;
-import org.apache.parquet.io.api.Binary;
-import org.apache.parquet.io.api.RecordConsumer;
+import org.apache.parquet.hadoop.BdecParquetReader;
+import org.apache.parquet.hadoop.BdecParquetWriter;
 import org.apache.parquet.schema.MessageType;
-import org.apache.parquet.schema.PrimitiveType;
 
 /**
  * Converts {@link ChannelData} buffered in {@link RowBuffer} to the Parquet format for faster
@@ -40,23 +24,36 @@
 public class ParquetFlusher implements Flusher<ParquetChunkData> {
   private static final Logging logger = new Logging(ParquetFlusher.class);
   private final MessageType schema;
+  private final boolean enableParquetInternalBuffering;
 
-  /** Construct parquet flusher from its schema. */
-  public ParquetFlusher(MessageType schema) {
+  /**
+   * Construct parquet flusher from its schema and set flag that indicates whether Parquet memory
+   * optimization is enabled, i.e. rows will be buffered in internal Parquet buffer.
+   */
+  public ParquetFlusher(MessageType schema, boolean enableParquetInternalBuffering) {
     this.schema = schema;
+    this.enableParquetInternalBuffering = enableParquetInternalBuffering;
   }
 
   @Override
   public SerializationResult serialize(
-      List<ChannelData<ParquetChunkData>> channelsDataPerTable,
-      ByteArrayOutputStream chunkData,
-      String filePath)
+      List<ChannelData<ParquetChunkData>> channelsDataPerTable, String filePath)
+      throws IOException {
+    if (enableParquetInternalBuffering) {
+      return serializeFromParquetWriteBuffers(channelsDataPerTable, filePath);
+    }
+    return serializeFromJavaObjects(channelsDataPerTable, filePath);
+  }
+
+  private SerializationResult serializeFromParquetWriteBuffers(
+      List<ChannelData<ParquetChunkData>> channelsDataPerTable, String filePath)
       throws IOException {
     List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
     long rowCount = 0L;
     String firstChannelFullyQualifiedTableName = null;
     Map<String, RowBufferStats> columnEpStatsMapCombined = null;
-    List<List<Object>> rows = null;
+    BdecParquetWriter mergedChannelWriter = null;
+    ByteArrayOutputStream mergedChunkData = new ByteArrayOutputStream();
     Pair<Long, Long> chunkMinMaxInsertTimeInMs = null;
 
     for (ChannelData<ParquetChunkData> data : channelsDataPerTable) {
@@ -77,9 +74,10 @@ public SerializationResult serialize(
           data.getBufferSize(),
           filePath);
 
-      if (rows == null) {
+      if (mergedChannelWriter == null) {
         columnEpStatsMapCombined = data.getColumnEps();
-        rows = new ArrayList<>();
+        mergedChannelWriter = data.getVectors().parquetWriter;
+        mergedChunkData = data.getVectors().output;
         firstChannelFullyQualifiedTableName = data.getChannelContext().getFullyQualifiedTableName();
         chunkMinMaxInsertTimeInMs = data.getMinMaxInsertTimeInMs();
       } else {
@@ -93,11 +91,13 @@ public SerializationResult serialize(
 
         columnEpStatsMapCombined =
             ChannelData.getCombinedColumnStatsMap(columnEpStatsMapCombined, data.getColumnEps());
+        data.getVectors().parquetWriter.close();
+        BdecParquetReader.readFileIntoWriter(
+            data.getVectors().output.toByteArray(), mergedChannelWriter);
         chunkMinMaxInsertTimeInMs =
             ChannelData.getCombinedMinMaxInsertTimeInMs(
                 chunkMinMaxInsertTimeInMs, data.getMinMaxInsertTimeInMs());
       }
-      rows.addAll(data.getVectors().rows);
 
       rowCount += data.getRowCount();
 
@@ -109,251 +109,94 @@ public SerializationResult serialize(
           filePath);
     }
 
-    Map<String, String> metadata = channelsDataPerTable.get(0).getVectors().metadata;
-
-    flushToParquetBdecChunk(chunkData, rows, metadata, channelsMetadataList);
+    if (mergedChannelWriter != null) {
+      mergedChannelWriter.close();
+    }
     return new SerializationResult(
-        channelsMetadataList, columnEpStatsMapCombined, rowCount, chunkMinMaxInsertTimeInMs);
+        channelsMetadataList,
+        columnEpStatsMapCombined,
+        rowCount,
+        mergedChunkData,
+        chunkMinMaxInsertTimeInMs);
   }
 
-  /**
-   * Flushes a parquet row chunk to the given BDEC output stream.
-   *
-   * @param bdecOutput BDEC output stream
-   * @param chunkRows chunk rows
-   * @param metadata chunk metadata
-   * @param channelsMetadataList metadata of the channels the chunk rows belong to
-   * @throws IOException thrown from Parquet library in case of writing problems
-   */
-  private void flushToParquetBdecChunk(
-      ByteArrayOutputStream bdecOutput,
-      List<List<Object>> chunkRows,
-      Map<String, String> metadata,
-      List<ChannelMetadata> channelsMetadataList)
+  private SerializationResult serializeFromJavaObjects(
+      List<ChannelData<ParquetChunkData>> channelsDataPerTable, String filePath)
       throws IOException {
-    try {
-      ParquetWriter<List<Object>> writer =
-          new BdecParquetWriterBuilder(bdecOutput, schema, metadata, channelsMetadataList)
-              // PARQUET_2_0 uses Encoding.DELTA_BYTE_ARRAY for byte arrays (e.g. SF sb16)
-              // server side does not support it TODO: SNOW-657238
-              .withWriterVersion(ParquetProperties.WriterVersion.PARQUET_1_0)
-
-              // the dictionary encoding (Encoding.*_DICTIONARY) is not supported by server side
-              // scanner yet
-              .withDictionaryEncoding(false)
+    List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
+    long rowCount = 0L;
+    String firstChannelFullyQualifiedTableName = null;
+    Map<String, RowBufferStats> columnEpStatsMapCombined = null;
+    List<List<Object>> rows = null;
+    BdecParquetWriter parquetWriter;
+    ByteArrayOutputStream mergedData = new ByteArrayOutputStream();
+    Pair<Long, Long> chunkMinMaxInsertTimeInMs = null;
 
-              // Historically server side scanner supports only the case when the row number in all
-              // pages is the same.
-              // The quick fix is to effectively disable the page size/row limit
-              // to always have one page per chunk until server side is generalised.
-              .withPageSize((int) Constants.MAX_CHUNK_SIZE_IN_BYTES * 2)
-              .withPageRowCountLimit(chunkRows.size() + 1)
-              .enableValidation()
-              .withCompressionCodec(CompressionCodecName.GZIP)
-              .withWriteMode(ParquetFileWriter.Mode.CREATE)
+    for (ChannelData<ParquetChunkData> data : channelsDataPerTable) {
+      // Create channel metadata
+      ChannelMetadata channelMetadata =
+          ChannelMetadata.builder()
+              .setOwningChannelFromContext(data.getChannelContext())
+              .setRowSequencer(data.getRowSequencer())
+              .setOffsetToken(data.getOffsetToken())
               .build();
+      // Add channel metadata to the metadata list
+      channelsMetadataList.add(channelMetadata);
 
-      // We can use lower level column writers and custom ValuesWriterFactory that uses plain byte
-      // array encoding used by PARQUET_1_0 and supported by server side
-      // TODO: SNOW-672143
-      for (List<Object> row : chunkRows) {
-        writer.write(row);
-      }
-      writer.close();
-    } catch (Throwable t) {
-      logger.logError("Parquet Flusher: failed to write", t);
-      throw t;
-    }
-  }
-
-  /**
-   * A parquet specific write builder.
-   *
-   * <p>This class is implemented as parquet library API requires, mostly to provide {@link
-   * BdecWriteSupport} implementation.
-   */
-  private static class BdecParquetWriterBuilder
-      extends ParquetWriter.Builder<List<Object>, BdecParquetWriterBuilder> {
-    private final MessageType schema;
-    private final Map<String, String> extraMetaData;
-    private final List<ChannelMetadata> channelsMetadataList;
-
-    protected BdecParquetWriterBuilder(
-        ByteArrayOutputStream stream,
-        MessageType schema,
-        Map<String, String> extraMetaData,
-        List<ChannelMetadata> channelsMetadataList) {
-      super(new ByteArrayOutputFile(stream));
-      this.schema = schema;
-      this.extraMetaData = extraMetaData;
-      this.channelsMetadataList = channelsMetadataList;
-    }
-
-    @Override
-    protected BdecParquetWriterBuilder self() {
-      return this;
-    }
-
-    @Override
-    protected WriteSupport<List<Object>> getWriteSupport(Configuration conf) {
-      return new BdecWriteSupport(schema, extraMetaData, channelsMetadataList);
-    }
-  }
-
-  /**
-   * A parquet specific file output implementation.
-   *
-   * <p>This class is implemented as parquet library API requires, mostly to create our {@link
-   * ByteArrayDelegatingPositionOutputStream} implementation.
-   */
-  private static class ByteArrayOutputFile implements OutputFile {
-    private final ByteArrayOutputStream stream;
-
-    private ByteArrayOutputFile(ByteArrayOutputStream stream) {
-      this.stream = stream;
-    }
-
-    @Override
-    public PositionOutputStream create(long blockSizeHint) throws IOException {
-      stream.reset();
-      return new ByteArrayDelegatingPositionOutputStream(stream);
-    }
-
-    @Override
-    public PositionOutputStream createOrOverwrite(long blockSizeHint) throws IOException {
-      return create(blockSizeHint);
-    }
-
-    @Override
-    public boolean supportsBlockSize() {
-      return false;
-    }
-
-    @Override
-    public long defaultBlockSize() {
-      return (int) MAX_CHUNK_SIZE_IN_BYTES;
-    }
-  }
-
-  /**
-   * A parquet specific output stream implementation.
-   *
-   * <p>This class is implemented as parquet library API requires, mostly to wrap our BDEC output
-   * {@link ByteArrayOutputStream}.
-   */
-  private static class ByteArrayDelegatingPositionOutputStream
-      extends DelegatingPositionOutputStream {
-    private final ByteArrayOutputStream stream;
-
-    public ByteArrayDelegatingPositionOutputStream(ByteArrayOutputStream stream) {
-      super(stream);
-      this.stream = stream;
-    }
+      logger.logDebug(
+          "Parquet Flusher: Start building channel={}, rowCount={}, bufferSize={} in blob={},"
+              + " enableParquetMemoryOptimization={}",
+          data.getChannelContext().getFullyQualifiedName(),
+          data.getRowCount(),
+          data.getBufferSize(),
+          filePath,
+          enableParquetInternalBuffering);
 
-    @Override
-    public long getPos() {
-      return stream.size();
-    }
-  }
+      if (rows == null) {
+        columnEpStatsMapCombined = data.getColumnEps();
+        rows = new ArrayList<>();
+        firstChannelFullyQualifiedTableName = data.getChannelContext().getFullyQualifiedTableName();
+        chunkMinMaxInsertTimeInMs = data.getMinMaxInsertTimeInMs();
+      } else {
+        // This method assumes that channelsDataPerTable is grouped by table. We double check
+        // here and throw an error if the assumption is violated
+        if (!data.getChannelContext()
+            .getFullyQualifiedTableName()
+            .equals(firstChannelFullyQualifiedTableName)) {
+          throw new SFException(ErrorCode.INVALID_DATA_IN_CHUNK);
+        }
 
-  /**
-   * A parquet specific write support implementation.
-   *
-   * <p>This class is implemented as parquet library API requires, mostly to serialize user column
-   * values depending on type into Parquet {@link RecordConsumer} in {@link
-   * BdecWriteSupport#write(List)}.
-   */
-  private static class BdecWriteSupport extends WriteSupport<List<Object>> {
-    MessageType schema;
-    RecordConsumer recordConsumer;
-    Map<String, String> extraMetadata;
-    List<ChannelMetadata> channelsMetadataList;
+        columnEpStatsMapCombined =
+            ChannelData.getCombinedColumnStatsMap(columnEpStatsMapCombined, data.getColumnEps());
+        chunkMinMaxInsertTimeInMs =
+            ChannelData.getCombinedMinMaxInsertTimeInMs(
+                chunkMinMaxInsertTimeInMs, data.getMinMaxInsertTimeInMs());
+      }
 
-    // TODO SNOW-672156: support specifying encodings and compression
-    BdecWriteSupport(
-        MessageType schema,
-        Map<String, String> extraMetadata,
-        List<ChannelMetadata> channelsMetadataList) {
-      this.schema = schema;
-      this.extraMetadata = extraMetadata;
-      this.channelsMetadataList = channelsMetadataList;
-    }
+      rows.addAll(data.getVectors().rows);
 
-    @Override
-    public WriteContext init(Configuration config) {
-      return new WriteContext(schema, extraMetadata);
-    }
+      rowCount += data.getRowCount();
 
-    @Override
-    public void prepareForWrite(RecordConsumer recordConsumer) {
-      this.recordConsumer = recordConsumer;
+      logger.logDebug(
+          "Parquet Flusher: Finish building channel={}, rowCount={}, bufferSize={} in blob={},"
+              + " enableParquetMemoryOptimization={}",
+          data.getChannelContext().getFullyQualifiedName(),
+          data.getRowCount(),
+          data.getBufferSize(),
+          filePath);
     }
 
-    @Override
-    public void write(List<Object> values) {
-      List<ColumnDescriptor> cols = schema.getColumns();
-      if (values.size() != cols.size()) {
-        List<String> channelNames =
-            this.channelsMetadataList.stream()
-                .map(ChannelMetadata::getChannelName)
-                .collect(Collectors.toList());
-        throw new ParquetEncodingException(
-            "Invalid input data in channels "
-                + channelNames
-                + ". Expecting "
-                + cols.size()
-                + " columns. Input had "
-                + values.size()
-                + " columns ("
-                + cols
-                + ") : "
-                + values);
-      }
+    Map<String, String> metadata = channelsDataPerTable.get(0).getVectors().metadata;
+    parquetWriter =
+        new BdecParquetWriter(mergedData, schema, metadata, firstChannelFullyQualifiedTableName);
+    rows.forEach(parquetWriter::writeRow);
+    parquetWriter.close();
 
-      recordConsumer.startMessage();
-      for (int i = 0; i < cols.size(); ++i) {
-        Object val = values.get(i);
-        // val.length() == 0 indicates a NULL value.
-        if (val != null) {
-          String fieldName = cols.get(i).getPath()[0];
-          recordConsumer.startField(fieldName, i);
-          PrimitiveType.PrimitiveTypeName typeName =
-              cols.get(i).getPrimitiveType().getPrimitiveTypeName();
-          switch (typeName) {
-            case BOOLEAN:
-              recordConsumer.addBoolean((boolean) val);
-              break;
-            case FLOAT:
-              recordConsumer.addFloat((float) val);
-              break;
-            case DOUBLE:
-              recordConsumer.addDouble((double) val);
-              break;
-            case INT32:
-              recordConsumer.addInteger((int) val);
-              break;
-            case INT64:
-              recordConsumer.addLong((long) val);
-              break;
-            case BINARY:
-              Binary binVal =
-                  val instanceof String
-                      ? Binary.fromString((String) val)
-                      : Binary.fromConstantByteArray((byte[]) val);
-              recordConsumer.addBinary(binVal);
-              break;
-            case FIXED_LEN_BYTE_ARRAY:
-              Binary binary = Binary.fromConstantByteArray((byte[]) val);
-              recordConsumer.addBinary(binary);
-              break;
-            default:
-              throw new ParquetEncodingException(
-                  "Unsupported column type: " + cols.get(i).getPrimitiveType());
-          }
-          recordConsumer.endField(fieldName, i);
-        }
-      }
-      recordConsumer.endMessage();
-    }
+    return new SerializationResult(
+        channelsMetadataList,
+        columnEpStatsMapCombined,
+        rowCount,
+        mergedData,
+        chunkMinMaxInsertTimeInMs);
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 9943c349e..f8bb85453 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -4,6 +4,8 @@
 
 package net.snowflake.ingest.streaming.internal;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
@@ -18,10 +20,13 @@
 import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.SFException;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.hadoop.BdecParquetWriter;
 import org.apache.parquet.schema.MessageType;
 import org.apache.parquet.schema.PrimitiveType;
 import org.apache.parquet.schema.Type;
@@ -36,24 +41,39 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
   private static final String PARQUET_MESSAGE_TYPE_NAME = "bdec";
 
   private final Map<String, Pair<ColumnMetadata, Integer>> fieldIndex;
+
+  /* map that contains metadata like typeinfo for columns and other information needed by the server scanner */
   private final Map<String, String> metadata;
+
+  /* Unflushed rows as Java objects. Needed for the Parquet w/o memory optimization. */
   private final List<List<Object>> data;
+  /* BDEC Parquet writer. It is used to buffer unflushed data in Parquet internal buffers instead of using Java objects */
+  private BdecParquetWriter bdecParquetWriter;
+
+  private ByteArrayOutputStream fileOutput;
   private final List<List<Object>> tempData;
+  private final String channelName;
 
   private MessageType schema;
-
+  private final boolean bufferForTests;
+  private final boolean enableParquetInternalBuffering;
   /** Construct a ParquetRowBuffer object. */
   ParquetRowBuffer(
       OpenChannelRequest.OnErrorOption onErrorOption,
       BufferAllocator allocator,
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
-      ChannelRuntimeState channelRuntimeState) {
+      ChannelRuntimeState channelRuntimeState,
+      boolean bufferForTests,
+      boolean enableParquetInternalBuffering) {
     super(onErrorOption, allocator, fullyQualifiedChannelName, rowSizeMetric, channelRuntimeState);
     fieldIndex = new HashMap<>();
     metadata = new HashMap<>();
     data = new ArrayList<>();
     tempData = new ArrayList<>();
+    channelName = fullyQualifiedChannelName;
+    this.bufferForTests = bufferForTests;
+    this.enableParquetInternalBuffering = enableParquetInternalBuffering;
   }
 
   @Override
@@ -86,6 +106,24 @@ public void setupSchema(List<ColumnMetadata> columns) {
       id++;
     }
     schema = new MessageType(PARQUET_MESSAGE_TYPE_NAME, parquetTypes);
+    createFileWriter();
+    tempData.clear();
+    data.clear();
+  }
+
+  /** Create BDEC file writer if Parquet memory optimization is enabled. */
+  private void createFileWriter() {
+    fileOutput = new ByteArrayOutputStream();
+    try {
+      if (enableParquetInternalBuffering) {
+        bdecParquetWriter = new BdecParquetWriter(fileOutput, schema, metadata, channelName);
+      } else {
+        this.bdecParquetWriter = null;
+      }
+      data.clear();
+    } catch (IOException e) {
+      throw new SFException(ErrorCode.INTERNAL_ERROR, "cannot create parquet writer", e);
+    }
   }
 
   @Override
@@ -99,7 +137,15 @@ float addRow(
       int curRowIndex,
       Map<String, RowBufferStats> statsMap,
       Set<String> formattedInputColumnNames) {
-    return addRow(row, data, statsMap, formattedInputColumnNames);
+    return addRow(row, this::writeRow, statsMap, formattedInputColumnNames);
+  }
+
+  void writeRow(List<Object> row) {
+    if (enableParquetInternalBuffering) {
+      bdecParquetWriter.writeRow(row);
+    } else {
+      data.add(row);
+    }
   }
 
   @Override
@@ -108,7 +154,7 @@ float addTempRow(
       int curRowIndex,
       Map<String, RowBufferStats> statsMap,
       Set<String> formattedInputColumnNames) {
-    return addRow(row, tempData, statsMap, formattedInputColumnNames);
+    return addRow(row, this::writeRow, statsMap, formattedInputColumnNames);
   }
 
   /**
@@ -122,7 +168,7 @@ float addTempRow(
    */
   private float addRow(
       Map<String, Object> row,
-      List<List<Object>> out,
+      Consumer<List<Object>> out,
       Map<String, RowBufferStats> statsMap,
       Set<String> inputColumnNames) {
     Object[] indexedRow = new Object[fieldIndex.size()];
@@ -142,7 +188,7 @@ private float addRow(
       indexedRow[colIndex] = valueWithSize.getValue();
       size += valueWithSize.getSize();
     }
-    out.add(Arrays.asList(indexedRow));
+    out.accept(Arrays.asList(indexedRow));
 
     for (String columnName : Sets.difference(this.fieldIndex.keySet(), inputColumnNames)) {
       statsMap.get(columnName).incCurrentNullCount();
@@ -152,7 +198,7 @@ private float addRow(
 
   @Override
   void moveTempRowsToActualBuffer(int tempRowCount) {
-    data.addAll(tempData);
+    tempData.forEach(this::writeRow);
   }
 
   @Override
@@ -167,22 +213,26 @@ boolean hasColumns() {
 
   @Override
   Optional<ParquetChunkData> getSnapshot(final String filePath) {
-    List<List<Object>> oldData = new ArrayList<>();
-    data.forEach(r -> oldData.add(new ArrayList<>(r)));
-
     // We insert the filename in the file itself as metadata so that streams can work on replicated
     // mixed tables. For a more detailed discussion on the topic see SNOW-561447 and
     // http://go/streams-on-replicated-mixed-tables
     metadata.put(Constants.PRIMARY_FILE_ID_KEY, StreamingIngestUtils.getShortname(filePath));
 
-    return oldData.isEmpty()
+    List<List<Object>> oldData = new ArrayList<>();
+    if (!enableParquetInternalBuffering) {
+      data.forEach(r -> oldData.add(new ArrayList<>(r)));
+    }
+    return rowCount <= 0
         ? Optional.empty()
-        : Optional.of(new ParquetChunkData(oldData, metadata));
+        : Optional.of(new ParquetChunkData(oldData, bdecParquetWriter, fileOutput, metadata));
   }
 
   /** Used only for testing. */
   @Override
   Object getVectorValueAt(String column, int index) {
+    if (data == null) {
+      return null;
+    }
     int colIndex = fieldIndex.get(column).getSecond();
     Object value = data.get(index).get(colIndex);
     ColumnMetadata columnMetadata = fieldIndex.get(column).getFirst();
@@ -215,20 +265,25 @@ int getTempRowCount() {
   @Override
   void reset() {
     super.reset();
+    createFileWriter();
     data.clear();
   }
 
+  /** Close the row buffer by releasing its internal resources. */
   @Override
-  public void close(String name) {
+  void closeInternal() {
     this.fieldIndex.clear();
-    logger.logInfo(
-        "Trying to close parquet buffer for channel={} from function={}",
-        channelFullyQualifiedName,
-        name);
+    if (bdecParquetWriter != null) {
+      try {
+        bdecParquetWriter.close();
+      } catch (IOException e) {
+        throw new SFException(ErrorCode.INTERNAL_ERROR, "Failed to close parquet writer", e);
+      }
+    }
   }
 
   @Override
   public Flusher<ParquetChunkData> createFlusher() {
-    return new ParquetFlusher(schema);
+    return new ParquetFlusher(schema, enableParquetInternalBuffering);
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 8eae060cd..349721ac9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -22,6 +22,7 @@
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.ParameterProvider;
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.Utils;
 import org.apache.arrow.memory.BufferAllocator;
@@ -120,7 +121,11 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
             bdecVersion,
             getFullyQualifiedName(),
             this::collectRowSize,
-            channelState);
+            channelState,
+            false,
+            owningClient != null
+                ? owningClient.getParameterProvider().getEnableParquetInternalBuffering()
+                : ParameterProvider.ENABLE_PARQUET_INTERNAL_BUFFERING_DEFAULT);
     logger.logInfo(
         "Channel={} created for table={}",
         this.channelFlushContext.getName(),
diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
index 85cb22575..acfa8e7bb 100644
--- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
+++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -21,6 +21,8 @@ public class ParameterProvider {
   public static final String BLOB_UPLOAD_MAX_RETRY_COUNT =
       "BLOB_UPLOAD_MAX_RETRY_COUNT".toLowerCase();
   public static final String MAX_MEMORY_LIMIT_IN_BYTES = "MAX_MEMORY_LIMIT_IN_BYTES".toLowerCase();
+  public static final String ENABLE_PARQUET_INTERNAL_BUFFERING =
+      "ENABLE_PARQUET_INTERNAL_BUFFERING".toLowerCase();
 
   // Default values
   public static final long BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT = 1000;
@@ -33,6 +35,10 @@ public class ParameterProvider {
   public static final int BLOB_UPLOAD_MAX_RETRY_COUNT_DEFAULT = 24;
   public static final long MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT = -1L;
 
+  /* Parameter that enables using internal Parquet buffers for buffering of rows before serializing.
+  It reduces memory consumption compared to using Java Objects for buffering.*/
+  public static final boolean ENABLE_PARQUET_INTERNAL_BUFFERING_DEFAULT = false;
+
   /** Map of parameter name to parameter value. This will be set by client/configure API Call. */
   private final Map<String, Object> parameterMap = new HashMap<>();
 
@@ -113,6 +119,12 @@ private void setParameterMap(Map<String, Object> parameterOverrides, Properties
 
     this.updateValue(
         MAX_MEMORY_LIMIT_IN_BYTES, MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT, parameterOverrides, props);
+
+    this.updateValue(
+        ENABLE_PARQUET_INTERNAL_BUFFERING,
+        ENABLE_PARQUET_INTERNAL_BUFFERING_DEFAULT,
+        parameterOverrides,
+        props);
   }
 
   /** @return Longest interval in milliseconds between buffer flushes */
@@ -219,6 +231,14 @@ public long getMaxMemoryLimitInBytes() {
     return (val instanceof String) ? Long.parseLong(val.toString()) : (long) val;
   }
 
+  /** @return Return whether memory optimization for Parquet is enabled. */
+  public boolean getEnableParquetInternalBuffering() {
+    Object val =
+        this.parameterMap.getOrDefault(
+            ENABLE_PARQUET_INTERNAL_BUFFERING, ENABLE_PARQUET_INTERNAL_BUFFERING_DEFAULT);
+    return (val instanceof String) ? Boolean.parseBoolean(val.toString()) : (boolean) val;
+  }
+
   @Override
   public String toString() {
     return "ParameterProvider{" + "parameterMap=" + parameterMap + '}';
diff --git a/src/main/java/org/apache/parquet/hadoop/BdecParquetReader.java b/src/main/java/org/apache/parquet/hadoop/BdecParquetReader.java
new file mode 100644
index 000000000..1a92a8cd4
--- /dev/null
+++ b/src/main/java/org/apache/parquet/hadoop/BdecParquetReader.java
@@ -0,0 +1,239 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package org.apache.parquet.hadoop;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.ParquetReadOptions;
+import org.apache.parquet.hadoop.api.InitContext;
+import org.apache.parquet.hadoop.api.ReadSupport;
+import org.apache.parquet.io.DelegatingSeekableInputStream;
+import org.apache.parquet.io.InputFile;
+import org.apache.parquet.io.SeekableInputStream;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.api.Converter;
+import org.apache.parquet.io.api.GroupConverter;
+import org.apache.parquet.io.api.PrimitiveConverter;
+import org.apache.parquet.io.api.RecordMaterializer;
+import org.apache.parquet.schema.GroupType;
+import org.apache.parquet.schema.MessageType;
+
+/**
+ * BDEC specific parquet reader.
+ *
+ * <p>Resides in parquet package because, it uses {@link InternalParquetRecordReader} that is
+ * package private.
+ */
+public class BdecParquetReader implements AutoCloseable {
+  private final InternalParquetRecordReader<List<Object>> reader;
+
+  /**
+   * @param data buffer where the data that has to be read resides.
+   * @throws IOException
+   */
+  public BdecParquetReader(byte[] data) throws IOException {
+    ParquetReadOptions options = ParquetReadOptions.builder().build();
+    ParquetFileReader fileReader = ParquetFileReader.open(new BdecInputFile(data), options);
+    reader = new InternalParquetRecordReader<>(new BdecReadSupport(), options.getRecordFilter());
+    reader.initialize(fileReader, options);
+  }
+
+  /**
+   * Reads the current row, i.e. list of values.
+   *
+   * @return current row
+   * @throws IOException
+   */
+  public List<Object> read() throws IOException {
+    try {
+      return reader.nextKeyValue() ? reader.getCurrentValue() : null;
+    } catch (InterruptedException e) {
+      throw new IOException(e);
+    }
+  }
+
+  /**
+   * Close the reader.
+   *
+   * @throws IOException
+   */
+  @Override
+  public void close() throws IOException {
+    reader.close();
+  }
+
+  /**
+   * Reads the input data using Parquet reader and writes them using a Parquet Writer.
+   *
+   * @param data input data to be read first and then written with outputWriter
+   * @param outputWriter output parquet writer
+   */
+  public static void readFileIntoWriter(byte[] data, BdecParquetWriter outputWriter) {
+    try (BdecParquetReader reader = new BdecParquetReader(data)) {
+      for (List<Object> record = reader.read(); record != null; record = reader.read()) {
+        outputWriter.writeRow(record);
+      }
+    } catch (IOException e) {
+      throw new SFException(ErrorCode.INTERNAL_ERROR, "Failed to merge parquet files", e);
+    }
+  }
+
+  private static class BdecInputFile implements InputFile {
+    private final byte[] data;
+
+    private BdecInputFile(byte[] data) {
+      this.data = data;
+    }
+
+    @Override
+    public long getLength() {
+      return data.length;
+    }
+
+    @Override
+    public SeekableInputStream newStream() {
+      return new BdecSeekableInputStream(new BdecByteArrayInputStream(data));
+    }
+  }
+
+  private static class BdecSeekableInputStream extends DelegatingSeekableInputStream {
+    private final BdecByteArrayInputStream stream;
+
+    public BdecSeekableInputStream(BdecByteArrayInputStream stream) {
+      super(stream);
+      this.stream = stream;
+    }
+
+    @Override
+    public long getPos() {
+      return stream.getPos();
+    }
+
+    @Override
+    public void seek(long newPos) {
+      stream.seek(newPos);
+    }
+  }
+
+  private static class BdecByteArrayInputStream extends ByteArrayInputStream {
+    public BdecByteArrayInputStream(byte[] buf) {
+      super(buf);
+    }
+
+    long getPos() {
+      return pos;
+    }
+
+    void seek(long newPos) {
+      pos = (int) newPos;
+    }
+  }
+
+  private static class BdecReadSupport extends ReadSupport<List<Object>> {
+    @Override
+    public RecordMaterializer<List<Object>> prepareForRead(
+        Configuration conf, Map<String, String> metaData, MessageType schema, ReadContext context) {
+      return new BdecRecordMaterializer(schema);
+    }
+
+    @Override
+    public ReadContext init(InitContext context) {
+      return new ReadContext(context.getFileSchema());
+    }
+  }
+
+  private static class BdecRecordMaterializer extends RecordMaterializer<List<Object>> {
+    public final BdecRecordConverter root;
+
+    public BdecRecordMaterializer(MessageType schema) {
+      this.root = new BdecRecordConverter(schema);
+    }
+
+    @Override
+    public List<Object> getCurrentRecord() {
+      return root.getCurrentRecord();
+    }
+
+    @Override
+    public GroupConverter getRootConverter() {
+      return root;
+    }
+  }
+
+  private static class BdecRecordConverter extends GroupConverter {
+    private final Converter[] converters;
+    private final int fieldNumber;
+    private Object[] record;
+
+    public BdecRecordConverter(GroupType schema) {
+      this.converters = new Converter[schema.getFieldCount()];
+      this.fieldNumber = schema.getFields().size();
+      for (int i = 0; i < fieldNumber; i++) {
+        converters[i] = new BdecPrimitiveConverter(i);
+      }
+    }
+
+    @Override
+    public Converter getConverter(int fieldIndex) {
+      return converters[fieldIndex];
+    }
+
+    List<Object> getCurrentRecord() {
+      return Arrays.asList(record);
+    }
+
+    @Override
+    public void start() {
+      record = new Object[fieldNumber];
+    }
+
+    @Override
+    public void end() {}
+
+    private class BdecPrimitiveConverter extends PrimitiveConverter {
+      protected final int index;
+
+      public BdecPrimitiveConverter(int index) {
+        this.index = index;
+      }
+
+      @Override
+      public void addBinary(Binary value) {
+        record[index] = value.getBytes();
+      }
+
+      @Override
+      public void addBoolean(boolean value) {
+        record[index] = value;
+      }
+
+      @Override
+      public void addDouble(double value) {
+        record[index] = value;
+      }
+
+      @Override
+      public void addFloat(float value) {
+        record[index] = value;
+      }
+
+      @Override
+      public void addInt(int value) {
+        record[index] = value;
+      }
+
+      @Override
+      public void addLong(long value) {
+        record[index] = value;
+      }
+    }
+  }
+}
diff --git a/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java b/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java
new file mode 100644
index 000000000..a353af7e3
--- /dev/null
+++ b/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java
@@ -0,0 +1,291 @@
+/*
+ * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
+ */
+
+package org.apache.parquet.hadoop;
+
+import static net.snowflake.ingest.utils.Constants.MAX_CHUNK_SIZE_IN_BYTES;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.parquet.column.ColumnDescriptor;
+import org.apache.parquet.column.ParquetProperties;
+import org.apache.parquet.column.values.factory.DefaultV1ValuesWriterFactory;
+import org.apache.parquet.crypto.FileEncryptionProperties;
+import org.apache.parquet.hadoop.api.WriteSupport;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.io.DelegatingPositionOutputStream;
+import org.apache.parquet.io.OutputFile;
+import org.apache.parquet.io.ParquetEncodingException;
+import org.apache.parquet.io.PositionOutputStream;
+import org.apache.parquet.io.api.Binary;
+import org.apache.parquet.io.api.RecordConsumer;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType;
+
+/**
+ * BDEC specific parquet writer.
+ *
+ * <p>Resides in parquet package because, it uses {@link InternalParquetRecordWriter} and {@link
+ * CodecFactory} that are package private.
+ */
+public class BdecParquetWriter implements AutoCloseable {
+  private final InternalParquetRecordWriter<List<Object>> writer;
+  private final CodecFactory codecFactory;
+
+  /**
+   * Creates a BDEC specific parquet writer.
+   *
+   * @param stream output
+   * @param schema row schema
+   * @param extraMetaData extra metadata
+   * @param channelName name of the channel that is using the writer
+   * @throws IOException
+   */
+  public BdecParquetWriter(
+      ByteArrayOutputStream stream,
+      MessageType schema,
+      Map<String, String> extraMetaData,
+      String channelName)
+      throws IOException {
+    OutputFile file = new ByteArrayOutputFile(stream);
+    ParquetProperties encodingProps = createParquetProperties();
+    Configuration conf = new Configuration();
+    WriteSupport<List<Object>> writeSupport =
+        new BdecWriteSupport(schema, extraMetaData, channelName);
+    WriteSupport.WriteContext writeContext = writeSupport.init(conf);
+
+    ParquetFileWriter fileWriter =
+        new ParquetFileWriter(
+            file,
+            schema,
+            ParquetFileWriter.Mode.CREATE,
+            ParquetWriter.DEFAULT_BLOCK_SIZE,
+            ParquetWriter.MAX_PADDING_SIZE_DEFAULT,
+            encodingProps.getColumnIndexTruncateLength(),
+            encodingProps.getStatisticsTruncateLength(),
+            encodingProps.getPageWriteChecksumEnabled(),
+            (FileEncryptionProperties) null);
+    fileWriter.start();
+
+    /*
+    Internally parquet writer initialises CodecFactory with the configured page size.
+    We set the page size to the max chunk size that is quite big in general.
+    CodecFactory allocates a byte buffer of that size on heap during initialisation.
+
+    If we use Parquet writer for buffering, there will be one writer per channel on each flush.
+    The memory will be allocated for each writer at the beginning even if we don't write anything with each writer,
+    which is the case when we enable parquet writer buffering.
+    Hence, to avoid huge memory allocations, we have to internally initialise CodecFactory with `ParquetWriter.DEFAULT_PAGE_SIZE` as it usually happens.
+    To get code access to this internal initialisation, we have to move the BdecParquetWriter class in the parquet.hadoop package.
+    */
+    codecFactory = new CodecFactory(conf, ParquetWriter.DEFAULT_PAGE_SIZE);
+    @SuppressWarnings("deprecation") // Parquet does not support the new one now
+    CodecFactory.BytesCompressor compressor = codecFactory.getCompressor(CompressionCodecName.GZIP);
+    writer =
+        new InternalParquetRecordWriter<>(
+            fileWriter,
+            writeSupport,
+            schema,
+            writeContext.getExtraMetaData(),
+            ParquetWriter.DEFAULT_BLOCK_SIZE,
+            compressor,
+            true,
+            encodingProps);
+  }
+
+  public void writeRow(List<Object> row) {
+    try {
+      writer.write(row);
+    } catch (InterruptedException | IOException e) {
+      throw new SFException(ErrorCode.INTERNAL_ERROR, "parquet row write failed", e);
+    }
+  }
+
+  @Override
+  public void close() throws IOException {
+    try {
+      writer.close();
+    } catch (InterruptedException e) {
+      throw new IOException(e);
+    } finally {
+      codecFactory.release();
+    }
+  }
+
+  private static ParquetProperties createParquetProperties() {
+    return ParquetProperties.builder()
+        // PARQUET_2_0 uses Encoding.DELTA_BYTE_ARRAY for byte arrays (e.g. SF sb16)
+        // server side does not support it TODO: SNOW-657238
+        .withWriterVersion(ParquetProperties.WriterVersion.PARQUET_1_0)
+        .withValuesWriterFactory(new DefaultV1ValuesWriterFactory())
+
+        // the dictionary encoding (Encoding.*_DICTIONARY) is not supported by server side
+        // scanner yet
+        .withDictionaryEncoding(false)
+
+        // Historically server side scanner supports only the case when the row number in all
+        // pages is the same.
+        // The quick fix is to effectively disable the page size/row limit
+        // to always have one page per chunk until server side is generalised.
+        .withPageSize((int) Constants.MAX_CHUNK_SIZE_IN_BYTES * 2)
+        .withPageRowCountLimit(Integer.MAX_VALUE)
+        .build();
+  }
+
+  /**
+   * A parquet specific file output implementation.
+   *
+   * <p>This class is implemented as parquet library API requires, mostly to create our {@link
+   * ByteArrayDelegatingPositionOutputStream} implementation.
+   */
+  private static class ByteArrayOutputFile implements OutputFile {
+    private final ByteArrayOutputStream stream;
+
+    private ByteArrayOutputFile(ByteArrayOutputStream stream) {
+      this.stream = stream;
+    }
+
+    @Override
+    public PositionOutputStream create(long blockSizeHint) throws IOException {
+      stream.reset();
+      return new ByteArrayDelegatingPositionOutputStream(stream);
+    }
+
+    @Override
+    public PositionOutputStream createOrOverwrite(long blockSizeHint) throws IOException {
+      return create(blockSizeHint);
+    }
+
+    @Override
+    public boolean supportsBlockSize() {
+      return false;
+    }
+
+    @Override
+    public long defaultBlockSize() {
+      return (int) MAX_CHUNK_SIZE_IN_BYTES;
+    }
+  }
+
+  /**
+   * A parquet specific output stream implementation.
+   *
+   * <p>This class is implemented as parquet library API requires, mostly to wrap our BDEC output
+   * {@link ByteArrayOutputStream}.
+   */
+  private static class ByteArrayDelegatingPositionOutputStream
+      extends DelegatingPositionOutputStream {
+    private final ByteArrayOutputStream stream;
+
+    public ByteArrayDelegatingPositionOutputStream(ByteArrayOutputStream stream) {
+      super(stream);
+      this.stream = stream;
+    }
+
+    @Override
+    public long getPos() {
+      return stream.size();
+    }
+  }
+
+  /**
+   * A parquet specific write support implementation.
+   *
+   * <p>This class is implemented as parquet library API requires, mostly to serialize user column
+   * values depending on type into Parquet {@link RecordConsumer} in {@link
+   * BdecWriteSupport#write(List)}.
+   */
+  private static class BdecWriteSupport extends WriteSupport<List<Object>> {
+    MessageType schema;
+    RecordConsumer recordConsumer;
+    Map<String, String> extraMetadata;
+    private final String channelName;
+
+    // TODO SNOW-672156: support specifying encodings and compression
+    BdecWriteSupport(MessageType schema, Map<String, String> extraMetadata, String channelName) {
+      this.schema = schema;
+      this.extraMetadata = extraMetadata;
+      this.channelName = channelName;
+    }
+
+    @Override
+    public WriteContext init(Configuration config) {
+      return new WriteContext(schema, extraMetadata);
+    }
+
+    @Override
+    public void prepareForWrite(RecordConsumer recordConsumer) {
+      this.recordConsumer = recordConsumer;
+    }
+
+    @Override
+    public void write(List<Object> values) {
+      List<ColumnDescriptor> cols = schema.getColumns();
+      if (values.size() != cols.size()) {
+        throw new ParquetEncodingException(
+            "Invalid input data in channel '"
+                + channelName
+                + "'. Expecting "
+                + cols.size()
+                + " columns. Input had "
+                + values.size()
+                + " columns ("
+                + cols
+                + ") : "
+                + values);
+      }
+
+      recordConsumer.startMessage();
+      for (int i = 0; i < cols.size(); ++i) {
+        Object val = values.get(i);
+        // val.length() == 0 indicates a NULL value.
+        if (val != null) {
+          String fieldName = cols.get(i).getPath()[0];
+          recordConsumer.startField(fieldName, i);
+          PrimitiveType.PrimitiveTypeName typeName =
+              cols.get(i).getPrimitiveType().getPrimitiveTypeName();
+          switch (typeName) {
+            case BOOLEAN:
+              recordConsumer.addBoolean((boolean) val);
+              break;
+            case FLOAT:
+              recordConsumer.addFloat((float) val);
+              break;
+            case DOUBLE:
+              recordConsumer.addDouble((double) val);
+              break;
+            case INT32:
+              recordConsumer.addInteger((int) val);
+              break;
+            case INT64:
+              recordConsumer.addLong((long) val);
+              break;
+            case BINARY:
+              Binary binVal =
+                  val instanceof String
+                      ? Binary.fromString((String) val)
+                      : Binary.fromConstantByteArray((byte[]) val);
+              recordConsumer.addBinary(binVal);
+              break;
+            case FIXED_LEN_BYTE_ARRAY:
+              Binary binary = Binary.fromConstantByteArray((byte[]) val);
+              recordConsumer.addBinary(binary);
+              break;
+            default:
+              throw new ParquetEncodingException(
+                  "Unsupported column type: " + cols.get(i).getPrimitiveType());
+          }
+          recordConsumer.endField(fieldName, i);
+        }
+      }
+      recordConsumer.endMessage();
+    }
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 6d25b11b3..96b2b6a07 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -29,16 +29,18 @@ public static Collection<Object[]> bdecVersion() {
     return Arrays.asList(
         new Object[][] {
           {"Arrow", Constants.BdecVersion.ONE},
-          {"Parquet", Constants.BdecVersion.THREE}
+          {"Parquet_w/o_optimization", Constants.BdecVersion.THREE}
         });
   }
 
   private final Constants.BdecVersion bdecVersion;
+  private final boolean enableParquetMemoryOptimization;
   private AbstractRowBuffer<?> rowBufferOnErrorContinue;
   private AbstractRowBuffer<?> rowBufferOnErrorAbort;
 
   public RowBufferTest(@SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
     this.bdecVersion = bdecVersion;
+    this.enableParquetMemoryOptimization = false;
   }
 
   @Before
@@ -116,7 +118,14 @@ static List<ColumnMetadata> createSchema() {
   private AbstractRowBuffer<?> createTestBuffer(OpenChannelRequest.OnErrorOption onErrorOption) {
     ChannelRuntimeState initialState = new ChannelRuntimeState("0", 0L, true);
     return AbstractRowBuffer.createRowBuffer(
-        onErrorOption, new RootAllocator(), bdecVersion, "test.buffer", rs -> {}, initialState);
+        onErrorOption,
+        new RootAllocator(),
+        bdecVersion,
+        "test.buffer",
+        rs -> {},
+        initialState,
+        true,
+        enableParquetMemoryOptimization);
   }
 
   @Test

From b9d123d91bbcfabd4f994d72a792c38884bc1f74 Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Wed, 25 Jan 2023 18:43:13 +0100
Subject: [PATCH 113/356] @no-snow fix parquet binary max length inconsistency
 with Arrow and do minor cleanup (#319)

---
 .../snowflake/ingest/streaming/internal/BlobBuilder.java   | 7 +------
 .../snowflake/ingest/streaming/internal/FlushService.java  | 7 +------
 .../ingest/streaming/internal/ParquetValueParser.java      | 2 +-
 .../snowflake/ingest/streaming/internal/RowBufferTest.java | 1 +
 4 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
index 2ae31c42d..9e34b30fd 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
@@ -63,15 +63,10 @@ class BlobBuilder {
    * @param blobData All the data for one blob. Assumes that all ChannelData in the inner List
    *     belongs to the same table. Will error if this is not the case
    * @param bdecVersion version of blob
-   * @param enableParquetMemoryOptimization indicates whether Parquet memory optimization should be
-   *     applied
    * @return {@link Blob} data
    */
   static <T> Blob constructBlobAndMetadata(
-      String filePath,
-      List<List<ChannelData<T>>> blobData,
-      Constants.BdecVersion bdecVersion,
-      boolean enableParquetMemoryOptimization)
+      String filePath, List<List<ChannelData<T>>> blobData, Constants.BdecVersion bdecVersion)
       throws IOException, NoSuchPaddingException, NoSuchAlgorithmException,
           InvalidAlgorithmParameterException, InvalidKeyException, IllegalBlockSizeException,
           BadPaddingException {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index e1c3d01e9..4ae49d077 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -442,12 +442,7 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
     Timer.Context buildContext = Utils.createTimerContext(this.owningClient.buildLatency);
 
     // Construct the blob along with the metadata of the blob
-    BlobBuilder.Blob blob =
-        BlobBuilder.constructBlobAndMetadata(
-            filePath,
-            blobData,
-            bdecVersion,
-            owningClient.getParameterProvider().getEnableParquetInternalBuffering());
+    BlobBuilder.Blob blob = BlobBuilder.constructBlobAndMetadata(filePath, blobData, bdecVersion);
     if (buildContext != null) {
       buildContext.stop();
     }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
index 1f5b1458e..dcf119204 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
@@ -345,7 +345,7 @@ private static String getBinaryValue(
    */
   private static byte[] getBinaryValueForLogicalBinary(
       Object value, RowBufferStats stats, ColumnMetadata columnMetadata) {
-    String maxLengthString = columnMetadata.getLength().toString();
+    String maxLengthString = columnMetadata.getByteLength().toString();
     byte[] bytes =
         DataValidationUtil.validateAndParseBinary(
             columnMetadata.getName(), value, Optional.of(maxLengthString).map(Integer::parseInt));
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 96b2b6a07..3538d7d7b 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -1070,6 +1070,7 @@ private void testE2EBinaryHelper(OpenChannelRequest.OnErrorOption onErrorOption)
     colBinary.setNullable(true);
     colBinary.setLogicalType("BINARY");
     colBinary.setLength(32);
+    colBinary.setByteLength(256);
     colBinary.setScale(0);
 
     innerBuffer.setupSchema(Collections.singletonList(colBinary));

From 0c6390de2e3b52f4acdb10f34e2dffe6916cec36 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 25 Jan 2023 17:45:05 -0800
Subject: [PATCH 114/356] SNOW-715618: Improve build latency for high
 throughput case with many channels (#321)

We're seeing some issue where the build time is high during high throughput case (20 million+ rows) with 10+ parallel channels going into the same table, and the root cause is that the buffer takes longer to transfer ownership and the next buffer already builds up with enough rows, so the file size becomes very big which is causing the compression time to be very high. In longer term, instead of checking the channel size, I think checking the allocator size per table would help this case.

In this PR, it contains a few changes to improve the build time:

Fix a wrong buffer size logic which causes the flush to be scheduled unexpectedly
Use parallel stream to get the buffer data
Reduce the channel size limit to schedule the flush faster
---
 .../streaming/internal/AbstractRowBuffer.java | 15 +++++----
 .../streaming/internal/FlushService.java      | 31 +++++++++++--------
 .../net/snowflake/ingest/utils/Constants.java |  4 +--
 3 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index bfd68b871..f07e8c03d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -274,7 +274,7 @@ Set<String> verifyInputColumns(
   @Override
   public InsertValidationResponse insertRows(
       Iterable<Map<String, Object>> rows, String offsetToken) {
-    float rowSize = 0F;
+    float rowsSizeInBytes = 0F;
     if (!hasColumns()) {
       throw new SFException(ErrorCode.INTERNAL_ERROR, "Empty column fields");
     }
@@ -290,9 +290,8 @@ public InsertValidationResponse insertRows(
               new InsertValidationResponse.InsertError(row, rowIndex);
           try {
             Set<String> inputColumnNames = verifyInputColumns(row, error);
-            rowSize += addRow(row, this.rowCount, this.statsMap, inputColumnNames);
+            rowsSizeInBytes += addRow(row, this.rowCount, this.statsMap, inputColumnNames);
             this.rowCount++;
-            this.bufferSize += rowSize;
           } catch (SFException e) {
             error.setException(e);
             response.addError(error);
@@ -308,22 +307,21 @@ public InsertValidationResponse insertRows(
         }
       } else {
         // If the on_error option is ABORT, simply throw the first exception
-        float tempRowSize = 0F;
+        float tempRowsSizeInBytes = 0F;
         int tempRowCount = 0;
         for (Map<String, Object> row : rows) {
           Set<String> inputColumnNames = verifyInputColumns(row, null);
-          tempRowSize += addTempRow(row, tempRowCount, this.tempStatsMap, inputColumnNames);
+          tempRowsSizeInBytes += addTempRow(row, tempRowCount, this.tempStatsMap, inputColumnNames);
           tempRowCount++;
         }
 
         moveTempRowsToActualBuffer(tempRowCount);
 
-        rowSize = tempRowSize;
+        rowsSizeInBytes = tempRowsSizeInBytes;
         if ((long) this.rowCount + tempRowCount >= Integer.MAX_VALUE) {
           throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
         }
         this.rowCount += tempRowCount;
-        this.bufferSize += rowSize;
         this.statsMap.forEach(
             (colName, stats) ->
                 this.statsMap.put(
@@ -331,8 +329,9 @@ public InsertValidationResponse insertRows(
                     RowBufferStats.getCombinedStats(stats, this.tempStatsMap.get(colName))));
       }
 
+      this.bufferSize += rowsSizeInBytes;
       this.channelState.setOffsetToken(offsetToken);
-      this.rowSizeMetric.accept(rowSize);
+      this.rowSizeMetric.accept(rowsSizeInBytes);
     } finally {
       this.tempStatsMap.values().forEach(RowBufferStats::reset);
       clearTempRows();
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 4ae49d077..b4b318a8c 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -19,6 +19,7 @@
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.Calendar;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -33,6 +34,7 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
 import javax.crypto.BadPaddingException;
 import javax.crypto.IllegalBlockSizeException;
 import javax.crypto.NoSuchPaddingException;
@@ -339,25 +341,28 @@ void distributeFlushTasks() {
 
     while (itr.hasNext()) {
       List<List<ChannelData<T>>> blobData = new ArrayList<>();
-      float totalBufferSize = 0;
+      AtomicReference<Float> totalBufferSizeInBytes = new AtomicReference<>((float) 0);
 
       final String filePath = getFilePath(this.targetStage.getClientPrefix());
 
       // Distribute work at table level, create a new blob if reaching the blob size limit
-      while (itr.hasNext() && totalBufferSize <= MAX_BLOB_SIZE_IN_BYTES) {
+      while (itr.hasNext() && totalBufferSizeInBytes.get() <= MAX_BLOB_SIZE_IN_BYTES) {
         ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<T>> table =
             itr.next().getValue();
-        List<ChannelData<T>> channelsDataPerTable = new ArrayList<>();
-        // TODO: we could do parallel stream to get the channelData if needed
-        for (SnowflakeStreamingIngestChannelInternal<T> channel : table.values()) {
-          if (channel.isValid()) {
-            ChannelData<T> data = channel.getData(filePath);
-            if (data != null) {
-              channelsDataPerTable.add(data);
-              totalBufferSize += data.getBufferSize();
-            }
-          }
-        }
+        List<ChannelData<T>> channelsDataPerTable = Collections.synchronizedList(new ArrayList<>());
+        // Use parallel stream since getData could be the performance bottleneck when we have a high
+        // number of channels
+        table.values().parallelStream()
+            .forEach(
+                channel -> {
+                  if (channel.isValid()) {
+                    ChannelData<T> data = channel.getData(filePath);
+                    if (data != null) {
+                      channelsDataPerTable.add(data);
+                      totalBufferSizeInBytes.updateAndGet(v -> v + data.getBufferSize());
+                    }
+                  }
+                });
         if (!channelsDataPerTable.isEmpty()) {
           blobData.add(channelsDataPerTable);
         }
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index 1440851b9..7d2903d60 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -34,8 +34,8 @@ public class Constants {
       7L; // Don't change, should match server side
   public static final int BLOB_UPLOAD_TIMEOUT_IN_SEC = 5;
   public static final int INSERT_THROTTLE_MAX_RETRY_COUNT = 60;
-  public static final long MAX_BLOB_SIZE_IN_BYTES = 512000000L;
-  public static final long MAX_CHUNK_SIZE_IN_BYTES = 32000000L;
+  public static final long MAX_BLOB_SIZE_IN_BYTES = 256000000L;
+  public static final long MAX_CHUNK_SIZE_IN_BYTES = 16000000L;
   public static final int BLOB_TAG_SIZE_IN_BYTES = 4;
   public static final int BLOB_VERSION_SIZE_IN_BYTES = 1;
   public static final int BLOB_FILE_SIZE_SIZE_IN_BYTES = 8;

From 4a977ffb34643ac40fd835a61c9733b13d03a536 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Thu, 26 Jan 2023 09:48:36 +0100
Subject: [PATCH 115/356] SNOW-663621 Fix problems with unicode strings (#300)

* SNOW-663621 Fix problems with unicode strings

Additionally the PR fixes the following closely related issues:

* SNOW-693446 Disable ingestion into collated strings
* SNOW-686944 Fix string comparison
* SNOW-682477 Use utf8 byte array length as string length

* Reject all collated columns

Co-authored-by: Toby Zhang <toby.zhang@snowflake.com>
---
 pom.xml                                       |   9 +-
 .../ingest/connection/RequestBuilder.java     |   2 +-
 .../streaming/internal/AbstractRowBuffer.java |  16 +-
 .../streaming/internal/ArrowRowBuffer.java    |   1 +
 .../streaming/internal/BinaryStringUtils.java |  40 ++
 .../internal/DataValidationUtil.java          |  56 ++-
 .../internal/FileColumnProperties.java        |  22 +-
 .../streaming/internal/ParquetRowBuffer.java  |   1 +
 .../streaming/internal/RowBufferStats.java    | 390 ++----------------
 .../net/snowflake/ingest/utils/ErrorCode.java |   3 +-
 .../ingest/ingest_error_messages.properties   |   3 +-
 .../java/net/snowflake/ingest/TestUtils.java  |  12 +
 .../internal/BinaryStringUtilsTest.java       | 107 +++++
 .../streaming/internal/ChannelDataTest.java   |  11 +-
 .../internal/DataValidationUtilTest.java      |  49 ++-
 .../internal/FileColumnPropertiesTest.java    |  30 ++
 .../streaming/internal/FlushServiceTest.java  |   1 -
 .../internal/ParquetValueParserTest.java      |  16 +-
 .../internal/RowBufferStatsTest.java          | 190 ++-------
 .../streaming/internal/RowBufferTest.java     |  52 ++-
 .../streaming/internal/StreamingIngestIT.java |  67 +--
 .../datatypes/AbstractDataTypeTest.java       |  32 +-
 .../internal/datatypes/StringsIT.java         | 183 +++++++-
 23 files changed, 616 insertions(+), 677 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/BinaryStringUtils.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/BinaryStringUtilsTest.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/FileColumnPropertiesTest.java

diff --git a/pom.xml b/pom.xml
index 45afd3932..a844c7ea4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
     <!-- Arifact name and version information -->
     <groupId>net.snowflake</groupId>
     <artifactId>snowflake-ingest-sdk</artifactId>
-    <version>1.0.2-beta.7</version>
+    <version>1.0.2-beta.8</version>
     <packaging>jar</packaging>
     <name>Snowflake Ingest SDK</name>
     <description>Snowflake Ingest SDK</description>
@@ -197,13 +197,6 @@
             <version>3.13.15</version>
         </dependency>
 
-        <!-- String collation-->
-        <dependency>
-            <groupId>com.ibm.icu</groupId>
-            <artifactId>icu4j</artifactId>
-            <version>70.1</version>
-        </dependency>
-
         <!-- jwt token for key pair authentication with GS -->
         <dependency>
             <groupId>com.nimbusds</groupId>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index bdb3e0321..3588883d4 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -127,7 +127,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.0.2-beta.7";
+  public static final String DEFAULT_VERSION = "1.0.2-beta.8";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index f07e8c03d..1e97b3593 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -191,7 +191,8 @@ public int getOrdinal() {
   }
 
   /**
-   * Adds non-nullable filed name.
+   * Adds non-nullable field name. It is used to check if all non-nullable fields have been
+   * provided.
    *
    * @param nonNullableFieldName non-nullable filed name
    */
@@ -199,6 +200,18 @@ void addNonNullableFieldName(String nonNullableFieldName) {
     nonNullableFieldNames.add(nonNullableFieldName);
   }
 
+  /** Throws an exception if the column has a collation defined. */
+  void validateColumnCollation(ColumnMetadata column) {
+    if (column.getCollation() != null) {
+      throw new SFException(
+          ErrorCode.UNSUPPORTED_DATA_TYPE,
+          String.format(
+              "Column %s with collation %s detected. Ingestion into collated columns is not"
+                  + " supported",
+              column.getName(), column.getCollation()));
+    }
+  }
+
   /**
    * Get the current buffer size
    *
@@ -221,6 +234,7 @@ public float getSize() {
    */
   Set<String> verifyInputColumns(
       Map<String, Object> row, InsertValidationResponse.InsertError error) {
+    // Map of unquoted column name -> original column name
     Map<String, String> inputColNamesMap =
         row.keySet().stream()
             .collect(Collectors.toMap(LiteralQuoteUtils::unquoteColumnName, value -> value));
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 34b924177..4a38c5d93 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -100,6 +100,7 @@ public void setupSchema(List<ColumnMetadata> columns) {
     List<FieldVector> tempVectors = new ArrayList<>();
 
     for (ColumnMetadata column : columns) {
+      validateColumnCollation(column);
       Field field = buildField(column);
       FieldVector vector = field.createVector(this.allocator);
       if (!field.isNullable()) {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BinaryStringUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/BinaryStringUtils.java
new file mode 100644
index 000000000..a518747b1
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BinaryStringUtils.java
@@ -0,0 +1,40 @@
+package net.snowflake.ingest.streaming.internal;
+
+import java.nio.ByteBuffer;
+import org.apache.commons.codec.binary.Hex;
+
+public class BinaryStringUtils {
+  private static final int MAX_LOB_LEN = 32;
+
+  /** Returns the number of unicode code points in a string */
+  static int unicodeCharactersCount(String s) {
+    return s.codePointCount(0, s.length());
+  }
+
+  /**
+   * Truncate an array of bytes to 32 bytes and optionally increment the last byte(s). More the one
+   * byte can be incremented in case it overflows.
+   */
+  static String truncateBytesAsHex(byte[] bytes, boolean truncateUp) {
+    if (bytes.length <= MAX_LOB_LEN) {
+      return Hex.encodeHexString(bytes);
+    }
+
+    // Round the least significant byte(s) up
+    if (truncateUp) {
+      int idx;
+      for (idx = MAX_LOB_LEN - 1; idx >= 0; idx--) {
+        // increment the current byte, if there was no overflow, we can stop
+        if (++bytes[idx] != 0) {
+          break;
+        }
+      }
+      // Whole prefix has overflown, return infinity
+      if (idx == -1) {
+        return "Z";
+      }
+    }
+
+    return Hex.encodeHexString(ByteBuffer.wrap(bytes, 0, MAX_LOB_LEN));
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 09e459de6..0591440f6 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -6,6 +6,7 @@
 
 import static net.snowflake.client.jdbc.internal.snowflake.common.core.SnowflakeDateTimeFormat.DATE;
 import static net.snowflake.client.jdbc.internal.snowflake.common.core.SnowflakeDateTimeFormat.TIMESTAMP;
+import static net.snowflake.ingest.streaming.internal.BinaryStringUtils.unicodeCharactersCount;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -14,6 +15,10 @@
 import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.nio.CharBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CodingErrorAction;
 import java.nio.charset.StandardCharsets;
 import java.time.Instant;
 import java.time.LocalDate;
@@ -90,8 +95,10 @@ private static SnowflakeDateTimeFormat createDateTimeFormatter() {
   private static JsonNode validateAndParseSemiStructuredAsJsonTree(
       String columnName, Object input, String snowflakeType) {
     if (input instanceof String) {
+      String stringInput = (String) input;
+      verifyValidUtf8(stringInput, columnName, snowflakeType);
       try {
-        return objectMapper.readTree((String) input);
+        return objectMapper.readTree(stringInput);
       } catch (JsonProcessingException e) {
         throw valueFormatNotAllowedException(columnName, input, snowflakeType, "Not a valid JSON");
       }
@@ -133,7 +140,7 @@ static String validateAndParseVariant(String columnName, Object input) {
     if (stringLength > MAX_SEMI_STRUCTURED_LENGTH) {
       throw valueFormatNotAllowedException(
           columnName,
-          input,
+          output,
           "VARIANT",
           String.format(
               "Variant too long: length=%d maxLength=%d",
@@ -266,7 +273,7 @@ static String validateAndParseArray(String columnName, Object input) {
     if (stringLength > MAX_SEMI_STRUCTURED_LENGTH) {
       throw valueFormatNotAllowedException(
           columnName,
-          jsonNode.toString(),
+          output,
           "ARRAY",
           String.format(
               "Array too large. length=%d maxLength=%d", stringLength, MAX_SEMI_STRUCTURED_LENGTH));
@@ -401,6 +408,7 @@ static String validateAndParseString(
     String output;
     if (input instanceof String) {
       output = (String) input;
+      verifyValidUtf8(output, columnName, "STRING");
     } else if (input instanceof Number) {
       output = new BigDecimal(input.toString()).stripTrailingZeros().toPlainString();
     } else if (input instanceof Boolean || input instanceof Character) {
@@ -412,15 +420,34 @@ static String validateAndParseString(
           "STRING",
           new String[] {"String", "Number", "boolean", "char"});
     }
-    int maxLength = maxLengthOptional.orElse(BYTES_16_MB);
+    byte[] utf8Bytes = output.getBytes(StandardCharsets.UTF_8);
 
-    if (output.length() > maxLength) {
+    // Strings can never be larger than 16MB
+    if (utf8Bytes.length > BYTES_16_MB) {
       throw valueFormatNotAllowedException(
           columnName,
           input,
           "STRING",
-          String.format("String too long: length=%d maxLength=%d", output.length(), maxLength));
+          String.format(
+              "String too long: length=%d bytes maxLength=%d bytes",
+              utf8Bytes.length, BYTES_16_MB));
     }
+
+    // If max allowed length is specified (e.g. VARCHAR(10)), the number of unicode characters must
+    // not exceed this value
+    maxLengthOptional.ifPresent(
+        maxAllowedCharacters -> {
+          int actualCharacters = unicodeCharactersCount(output);
+          if (actualCharacters > maxAllowedCharacters) {
+            throw valueFormatNotAllowedException(
+                columnName,
+                input,
+                "STRING",
+                String.format(
+                    "String too long: length=%d characters maxLength=%d characters",
+                    actualCharacters, maxAllowedCharacters));
+          }
+        });
     return output;
   }
 
@@ -784,4 +811,21 @@ private static SFTimestamp timestampFromInstant(Instant instant, TimeZone timeZo
     return SFTimestamp.fromNanoseconds(
         instant.getEpochSecond() * Power10.intTable[9] + instant.getNano(), timeZone);
   }
+
+  /**
+   * Validates that a string is valid UTF-8 string. It catches situations like unmatched high/low
+   * UTF-16 surrogate, for example.
+   */
+  private static void verifyValidUtf8(String input, String columnName, String dataType) {
+    CharsetEncoder charsetEncoder =
+        StandardCharsets.UTF_8
+            .newEncoder()
+            .onMalformedInput(CodingErrorAction.REPORT)
+            .onUnmappableCharacter(CodingErrorAction.REPORT);
+    try {
+      charsetEncoder.encode(CharBuffer.wrap(input));
+    } catch (CharacterCodingException e) {
+      throw valueFormatNotAllowedException(columnName, input, dataType, "Invalid Unicode string");
+    }
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java b/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java
index 581d96bbd..290248a83 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java
@@ -1,11 +1,14 @@
 package net.snowflake.ingest.streaming.internal;
 
+import static net.snowflake.ingest.streaming.internal.BinaryStringUtils.truncateBytesAsHex;
+
 import com.fasterxml.jackson.annotation.JsonProperty;
 import java.math.BigInteger;
 import java.util.Objects;
 
 /** Audit register endpoint/FileColumnPropertyDTO property list. */
 class FileColumnProperties {
+
   private String minStrValue;
 
   private String maxStrValue;
@@ -61,12 +64,21 @@ class FileColumnProperties {
             : stats.getCurrentMaxRealValue());
     this.setMaxLength(stats.getCurrentMaxLength());
 
-    // Collated and non-collated strings are intentionally equal here as required by Snowflake
-    this.setMaxStrNonCollated(stats.getCurrentMaxColStrValue());
-    this.setMinStrNonCollated(stats.getCurrentMinColStrValue());
+    this.setMaxStrNonCollated(null);
+    this.setMinStrNonCollated(null);
+
+    // current hex-encoded min value, truncated down to 32 bytes
+    if (stats.getCurrentMinStrValue() != null) {
+      String truncatedAsHex = truncateBytesAsHex(stats.getCurrentMinStrValue(), false);
+      this.setMinStrValue(truncatedAsHex);
+    }
+
+    // current hex-encoded max value, truncated up to 32 bytes
+    if (stats.getCurrentMaxStrValue() != null) {
+      String truncatedAsHex = truncateBytesAsHex(stats.getCurrentMaxStrValue(), true);
+      this.setMaxStrValue(truncatedAsHex);
+    }
 
-    this.setMaxStrValue(stats.getCurrentMaxColStrValue());
-    this.setMinStrValue(stats.getCurrentMinColStrValue());
     this.setNullCount(stats.getCurrentNullCount());
     this.setDistinctValues(stats.getDistinctValues());
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index f8bb85453..f1d0daa9a 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -87,6 +87,7 @@ public void setupSchema(List<ColumnMetadata> columns) {
     // precision
     int id = 1;
     for (ColumnMetadata column : columns) {
+      validateColumnCollation(column);
       ParquetTypeGenerator.ParquetTypeInfo typeInfo =
           ParquetTypeGenerator.generateColumnParquetTypeInfo(column, id);
       parquetTypes.add(typeInfo.getParquetType());
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
index 2a6503084..7be1c0050 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
@@ -4,299 +4,17 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import com.ibm.icu.lang.UCharacter;
-import com.ibm.icu.text.CollationKey;
-import com.ibm.icu.text.Collator;
-import com.ibm.icu.util.ULocale;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
 import java.util.Objects;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 
 /** Keeps track of the active EP stats, used to generate a file EP info */
 class RowBufferStats {
-  private static class CollationDefinition {
-    /** The name of this collation */
-    private String name;
 
-    /** Derived ICU collation name */
-    private String icuCollationString = null;
-
-    /** Defines if a given collation trims strings on the left side */
-    private boolean ltrim = false;
-
-    /** Defines if a given collation trims strings on the right side */
-    private boolean rtrim = false;
-
-    /** Defines if a given collation applies "lower" before comparison */
-    private boolean lower = false;
-
-    /** Defines if a given collation applies "upper" before comparison */
-    private boolean upper = false;
-
-    /** If true or null, the collation is case sensitive */
-    private Boolean caseSensitive = null;
-
-    /** If true or null, the collation is accent sensitive */
-    private Boolean accentSensitive = null;
-
-    /** If true or null, the collation is punctuation sensitive */
-    private Boolean punctuationSensitive = null;
-
-    /**
-     * Defines if a given collation is forced, i.e. goes through the full collation pipeline even if
-     * not strictly needed (i.e. UTF8)
-     */
-    private boolean forced = false;
-
-    private static String LANG_UTF8 = "utf8";
-    private static final String LANG_BIN = "bin";
-    private static final String LANG_EMPTY = "";
-
-    /** Language, e.g. "en_EN". LANG_UTF8 is the default */
-    private String language = LANG_UTF8;
-
-    private Collator collator;
-
-    /** Constructor - only called internally */
-    private CollationDefinition(String name) {
-      name = name.toLowerCase();
-      this.name = name;
-
-      // Empty collation is UTF8, treat it as such
-      if (name.isEmpty()) {
-        name = LANG_UTF8;
-      }
-
-      /** Language, e.g. "en_EN". LANG_UTF8 is the default */
-      String lang = LANG_UTF8;
-
-      // Variables used to construct the ICU string
-      boolean firstLower = false;
-      boolean firstLowerIsSet = false;
-
-      // Parse the collation as a sequence of "-" divided options
-      String[] elements = name.split("-");
-      assert (elements.length > 0); // empty string detected earlier
-
-      // Then other options
-      for (int i = 0; i < elements.length; i++) {
-        switch (elements[i]) {
-            // ICU options
-          case "ci":
-            caseSensitive = false;
-            break;
-          case "cs":
-            caseSensitive = true;
-            break;
-          case "ai":
-            accentSensitive = false;
-            break;
-          case "as":
-            accentSensitive = true;
-            break;
-          case "pi":
-            punctuationSensitive = false;
-            break;
-          case "ps":
-            punctuationSensitive = true;
-            break;
-          case "fu":
-            firstLower = false;
-            firstLowerIsSet = true;
-            break;
-          case "fl":
-            firstLower = true;
-            firstLowerIsSet = true;
-            break;
-            // Snowflake-specific options
-          case "lower":
-            lower = true;
-            break;
-          case "upper":
-            upper = true;
-            break;
-          case "rtrim":
-            rtrim = true;
-            break;
-          case "ltrim":
-            ltrim = true;
-            break;
-          case "trim":
-            ltrim = rtrim = true;
-            break;
-          case "forced":
-            forced = true;
-            break;
-          default:
-            // Unknown element can be the language identificator, if it's the first one
-            if (i == 0) {
-              language = elements[0];
-            } else {
-              throw new SFException(
-                  ErrorCode.INVALID_COLLATION_STRING, name, "Unknown option: " + elements[i]);
-            }
-        }
-      }
-
-      /* Sanity checks */
-      if (upper && lower)
-        throw new SFException(
-            ErrorCode.INVALID_COLLATION_STRING,
-            name,
-            "Options 'upper' and 'lower' are mutually exclusive");
-
-      if ((language == null
-          || language.equalsIgnoreCase(LANG_EMPTY)
-          || language.equalsIgnoreCase(LANG_UTF8)
-          || language.equalsIgnoreCase(LANG_BIN))) {
-        // Check we didn't provide any of the ICU-specific options
-        if (caseSensitive != null)
-          throw new SFException(
-              ErrorCode.INVALID_COLLATION_STRING,
-              name,
-              "Case sensitivity option not allowed for the UTF8 collation");
-        if (accentSensitive != null)
-          throw new SFException(
-              ErrorCode.INVALID_COLLATION_STRING,
-              name,
-              "Accent sensitivity option not allowed for the UTF8 collation");
-        if (firstLowerIsSet)
-          throw new SFException(
-              ErrorCode.INVALID_COLLATION_STRING,
-              name,
-              "Upper/lower preference option not allowed for the UTF8 collation");
-        if (punctuationSensitive != null)
-          throw new SFException(
-              ErrorCode.INVALID_COLLATION_STRING,
-              name,
-              "Punctuation sensitivity option not allowed for the UTF8 collation");
-      } else {
-        // Set the defaults
-        // NOTE: firstLower does not have a default value
-        if (caseSensitive == null) caseSensitive = true;
-        if (accentSensitive == null) accentSensitive = true;
-        if (punctuationSensitive == null) punctuationSensitive = true;
-
-        // Construct an ICU string, e.g. "de@colStrength=primary;colCaseFirst=lower"
-        // This string will be used by COLLATE_TO_BINARY and others.
-        // See http://userguide.icu-project.org/collation/concepts#TOC-Collator-naming-scheme
-
-        // We start with the language, and then construct a series of ";option=something".
-        // At the end, we'll switch the first ';' to '@'
-        String icu = language;
-
-        /*
-         * From https://www.unicode.org/reports/tr35/tr35-collation.html#Collation_Element :
-         * 3.4.1 Common settings combinations
-         * Some commonly used parametric collation settings are available via combinations of LDML settings attributes:
-         * - “Ignore accents”: strength=primary
-         * - “Ignore accents” but take case into account: strength=primary caseLevel=on
-         * - “Ignore case”: strength=secondary
-         */
-        if (!accentSensitive) {
-          icu += ";colStrength=primary";
-          // In ICU, "primary" implies case-insensitive.
-          // If we're case sensitive, force it with "colCaseLevel"
-          icu += ";colCaseLevel=" + (caseSensitive ? "yes" : "no");
-        } else if (!caseSensitive) {
-          icu += ";colStrength=secondary";
-        }
-        if (firstLowerIsSet) {
-          icu += ";colCaseFirst=" + (firstLower ? "lower" : "upper");
-        }
-        if (punctuationSensitive != null) {
-          icu += ";colAlternate=" + (punctuationSensitive ? "non-ignorable" : "shifted");
-        }
-
-        // Fix the first ";" to be "@"
-        this.icuCollationString = icu.replaceFirst(";", "@");
-      }
-      if (this.icuCollationString != null) {
-        ULocale locale = new ULocale(icuCollationString);
-        collator = Collator.getInstance(locale);
-      }
-    }
-
-    byte[] performConversion(String input) {
-      /*
-       * WARNING: the logic should be in sync with CollationEvaluator.cpp !!!
-       */
-      if (ltrim || rtrim) {
-        // Apply trim
-        int inputIdx = 0;
-        int inputLen = input.length();
-
-        if (ltrim) {
-          while (inputLen > 0 && input.charAt(inputIdx) == ' ') {
-            inputLen--;
-            inputIdx++;
-          }
-        }
-        if (rtrim) {
-          while (inputLen > 0 && input.charAt(inputIdx + inputLen - 1) == ' ') {
-            inputLen--;
-          }
-        }
-        input = input.substring(inputIdx, inputIdx + inputLen);
-      }
-
-      // Handle upper/lower
-      if (upper) {
-        input = UCharacter.toUpperCase(input);
-      }
-      if (lower) {
-        input = UCharacter.toLowerCase(input);
-      }
-
-      if (icuCollationString == null) {
-        // No need for ICU key generation, convert (modified) input to UTF-8 bytes
-        return input.getBytes(StandardCharsets.UTF_8);
-      }
-
-      // ICU collation is needed
-      // Perform the conversion
-      CollationKey cltKey = collator.getCollationKey(input);
-      byte[] cltKeyBytes = cltKey.toByteArray();
-      int cltKeySize = cltKeyBytes.length;
-
-      if (cltKeySize > 0) {
-        // Get rid of the trailing 0
-        if (cltKeyBytes[cltKeySize - 1] != 0) {
-          throw new SFException(
-              ErrorCode.INTERNAL_ERROR, "unexpected_non_zero_end_of_collated_key");
-        }
-        cltKeySize -= 1;
-
-        // Check if the produced key is all 0x01 - this can happen when we collate to an empty
-        // string,
-        // but the ICU still produces 0x01 dividers between various parts of the collation key.
-        // It's better to just treat these as an empty string.
-        int idx;
-        for (idx = 0; idx < cltKeySize; idx++) {
-          if (cltKeyBytes[idx] != 0x01) {
-            break;
-          }
-        }
-
-        if (idx == cltKeySize) {
-          // Mark as an empty string
-          cltKeySize = 0;
-        }
-
-        cltKeyBytes = Arrays.copyOf(cltKeyBytes, cltKeySize);
-      }
-      // Create an SFBinary
-      return cltKeyBytes;
-    }
-  }
-
-  private String currentMinColStrValue;
-  private String currentMaxColStrValue;
-  private byte[] currentMinColStrValueInBytes;
-  private byte[] currentMaxColStrValueInBytes;
+  private byte[] currentMinStrValue;
+  private byte[] currentMaxStrValue;
   private BigInteger currentMinIntValue;
   private BigInteger currentMaxIntValue;
   private Double currentMinRealValue;
@@ -304,8 +22,7 @@ byte[] performConversion(String input) {
   private long currentNullCount;
   // for binary or string columns
   private long currentMaxLength;
-  private CollationDefinition collationDefinition;
-  private final String collationDefinitionString;
+  private String collationDefinitionString;
   /** Display name is required for the registration endpoint */
   private final String columnDisplayName;
 
@@ -315,9 +32,6 @@ byte[] performConversion(String input) {
   RowBufferStats(String columnDisplayName, String collationDefinitionString) {
     this.columnDisplayName = columnDisplayName;
     this.collationDefinitionString = collationDefinitionString;
-    if (collationDefinitionString != null) {
-      this.collationDefinition = new CollationDefinition(collationDefinitionString);
-    }
     reset();
   }
 
@@ -326,10 +40,8 @@ byte[] performConversion(String input) {
   }
 
   void reset() {
-    this.currentMaxColStrValue = null;
-    this.currentMinColStrValue = null;
-    this.currentMaxColStrValueInBytes = null;
-    this.currentMinColStrValueInBytes = null;
+    this.currentMaxStrValue = null;
+    this.currentMinStrValue = null;
     this.currentMaxIntValue = null;
     this.currentMinIntValue = null;
     this.currentMaxRealValue = null;
@@ -338,13 +50,6 @@ void reset() {
     this.currentMaxLength = 0;
   }
 
-  byte[] getCollatedBytes(String value) {
-    if (collationDefinition != null) {
-      return collationDefinition.performConversion(value);
-    }
-    return value.getBytes(StandardCharsets.UTF_8);
-  }
-
   // TODO performance test this vs in place update
   static RowBufferStats getCombinedStats(RowBufferStats left, RowBufferStats right) {
     if (!Objects.equals(left.getCollationDefinitionString(), right.collationDefinitionString)) {
@@ -368,14 +73,14 @@ static RowBufferStats getCombinedStats(RowBufferStats left, RowBufferStats right
       combined.addIntValue(right.currentMaxIntValue);
     }
 
-    if (left.currentMinColStrValue != null) {
-      combined.addStrValue(left.currentMinColStrValue);
-      combined.addStrValue(left.currentMaxColStrValue);
+    if (left.currentMinStrValue != null) {
+      combined.addStrValue(left.currentMinStrValue);
+      combined.addStrValue(left.currentMaxStrValue);
     }
 
-    if (right.currentMinColStrValue != null) {
-      combined.addStrValue(right.currentMinColStrValue);
-      combined.addStrValue(right.currentMaxColStrValue);
+    if (right.currentMinStrValue != null) {
+      combined.addStrValue(right.currentMinStrValue);
+      combined.addStrValue(right.currentMaxStrValue);
     }
 
     if (left.currentMinRealValue != null) {
@@ -394,67 +99,34 @@ static RowBufferStats getCombinedStats(RowBufferStats left, RowBufferStats right
     return combined;
   }
 
-  void addStrValue(String inputValue) {
-    this.setCurrentMaxLength(inputValue.length());
-    String value =
-        inputValue.length() > MAX_LOB_LEN ? inputValue.substring(0, MAX_LOB_LEN) : inputValue;
+  void addStrValue(String value) {
+    addStrValue(value.getBytes(StandardCharsets.UTF_8));
+  }
 
-    byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
-    byte[] collatedValueBytes = getCollatedBytes(value);
+  void addStrValue(byte[] valueBytes) {
+    this.setCurrentMaxLength(valueBytes.length);
 
     // Check if new min/max string
-    if (this.currentMinColStrValue == null) {
-      this.currentMinColStrValue = value;
-      this.currentMinColStrValueInBytes = collatedValueBytes;
-
-      /*
-      Snowflake stores the first MAX_LOB_LEN characters of a string.
-      When truncating the max value, we increment the last max value
-      byte by one to ensure the max value stat is greater than the actual max value.
-       */
-      if (inputValue.length() > MAX_LOB_LEN) {
-        byte[] incrementedValueBytes = valueBytes.clone();
-        byte[] incrementedCollatedValueBytes = collatedValueBytes.clone();
-        incrementedValueBytes[MAX_LOB_LEN - 1]++;
-        incrementedCollatedValueBytes[MAX_LOB_LEN - 1]++;
-        this.currentMaxColStrValue = new String(incrementedValueBytes);
-        this.currentMaxColStrValueInBytes = incrementedCollatedValueBytes;
-      } else {
-        this.currentMaxColStrValue = value;
-        this.currentMaxColStrValueInBytes = collatedValueBytes;
-      }
+    if (this.currentMinStrValue == null) {
+      this.currentMinStrValue = valueBytes;
+      this.currentMaxStrValue = valueBytes;
     } else {
-      // Collated comparison
-      if (compare(currentMinColStrValueInBytes, collatedValueBytes) > 0) {
-        this.currentMinColStrValue = value;
-        this.currentMinColStrValueInBytes = collatedValueBytes;
-      } else if (compare(currentMaxColStrValueInBytes, collatedValueBytes) < 0) {
-        /*
-        Snowflake stores the first MAX_LOB_LEN characters of a string.
-        When truncating the max value, we increment the last max value
-        byte by one to ensure the max value stat is greater than the actual max value.
-         */
-        if (inputValue.length() > MAX_LOB_LEN) {
-          byte[] incrementedValueBytes = valueBytes.clone();
-          byte[] incrementedCollatedValueBytes = collatedValueBytes.clone();
-          incrementedValueBytes[MAX_LOB_LEN - 1]++;
-          incrementedCollatedValueBytes[MAX_LOB_LEN - 1]++;
-          this.currentMaxColStrValue = new String(incrementedValueBytes);
-          this.currentMaxColStrValueInBytes = incrementedCollatedValueBytes;
-        } else {
-          this.currentMaxColStrValue = value;
-          this.currentMaxColStrValueInBytes = collatedValueBytes;
-        }
+      // Check if the input is less than the existing min
+      if (compareUnsigned(currentMinStrValue, valueBytes) > 0) {
+        this.currentMinStrValue = valueBytes;
+        // Check if the input is more than the existing max
+      } else if (compareUnsigned(currentMaxStrValue, valueBytes) < 0) {
+        this.currentMaxStrValue = valueBytes;
       }
     }
   }
 
-  String getCurrentMinColStrValue() {
-    return currentMinColStrValue;
+  byte[] getCurrentMinStrValue() {
+    return currentMinStrValue;
   }
 
-  String getCurrentMaxColStrValue() {
-    return currentMaxColStrValue;
+  byte[] getCurrentMaxStrValue() {
+    return currentMaxStrValue;
   }
 
   void addIntValue(BigInteger value) {
@@ -546,12 +218,12 @@ String getColumnDisplayName() {
    *     second array; and a value greater than 0 if the first array is lexicographically greater
    *     than the second array
    */
-  static int compare(byte[] a, byte[] b) {
+  static int compareUnsigned(byte[] a, byte[] b) {
     if (a == b) return 0;
 
     for (int mismatchIdx = 0; mismatchIdx < Math.min(a.length, b.length); mismatchIdx++) {
       if (a[mismatchIdx] != b[mismatchIdx]) {
-        return Byte.compare(a[mismatchIdx], b[mismatchIdx]);
+        return Byte.toUnsignedInt(a[mismatchIdx]) - Byte.toUnsignedInt(b[mismatchIdx]);
       }
     }
 
diff --git a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
index 095d08d14..3b3c4f9f7 100644
--- a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
+++ b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
@@ -33,7 +33,8 @@ public enum ErrorCode {
   CHANNELS_WITH_UNCOMMITTED_ROWS("0025"),
   INVALID_COLLATION_STRING("0026"),
   ENCRYPTION_FAILURE("0027"),
-  CHANNEL_STATUS_INVALID("0028");
+  CHANNEL_STATUS_INVALID("0028"),
+  UNSUPPORTED_DATA_TYPE("0029");
 
   public static final String errorMessageResource = "net.snowflake.ingest.ingest_error_messages";
 
diff --git a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
index ef0149ce4..807486bfc 100644
--- a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
+++ b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
@@ -31,5 +31,4 @@
 0026=Invalid collation string: {0}. {1}
 0027=Failure during data encryption.
 0028=Get channel status indicates Channel {0} is invalid with status code {1}, please reopening the channel.
-
-
+0029=Data type not supported: {0}
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 96d46b393..9506e4fac 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -372,4 +372,16 @@ public static void waitForOffset(SnowflakeStreamingIngestChannel channel, String
             "Timeout exceeded while waiting for offset %s. Last committed offset: %s",
             expectedOffset, lastCommittedOffset));
   }
+
+  /**
+   * Creates a string from a certain number of concatenated strings e.g. buildString("ab", 2) =>
+   * abab
+   */
+  public static String buildString(String str, int count) {
+    StringBuilder sb = new StringBuilder(count);
+    for (int i = 0; i < count; i++) {
+      sb.append(str);
+    }
+    return sb.toString();
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/BinaryStringUtilsTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/BinaryStringUtilsTest.java
new file mode 100644
index 000000000..f836574b3
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/BinaryStringUtilsTest.java
@@ -0,0 +1,107 @@
+package net.snowflake.ingest.streaming.internal;
+
+import static net.snowflake.ingest.streaming.internal.BinaryStringUtils.truncateBytesAsHex;
+import static net.snowflake.ingest.streaming.internal.BinaryStringUtils.unicodeCharactersCount;
+
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class BinaryStringUtilsTest {
+
+  @Test
+  public void testUnicodeCodePointsCalculation() {
+    Assert.assertEquals(1, unicodeCharactersCount("🍞"));
+  }
+
+  @Test
+  public void testTruncation() throws DecoderException {
+    // Test empty input
+    Assert.assertEquals("", truncateBytesAsHex(new byte[0], false));
+    Assert.assertEquals("", truncateBytesAsHex(new byte[0], true));
+
+    // Test basic case
+    Assert.assertEquals("aa", truncateBytesAsHex(Hex.decodeHex("aa"), false));
+    Assert.assertEquals("aa", truncateBytesAsHex(Hex.decodeHex("aa"), true));
+
+    // Test exactly 32 bytes
+    Assert.assertEquals(
+        "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+        truncateBytesAsHex(
+            Hex.decodeHex("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
+            false));
+    Assert.assertEquals(
+        "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+        truncateBytesAsHex(
+            Hex.decodeHex("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
+            true));
+
+    Assert.assertEquals(
+        "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+        truncateBytesAsHex(
+            Hex.decodeHex("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
+            false));
+
+    Assert.assertEquals(
+        "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+        truncateBytesAsHex(
+            Hex.decodeHex("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
+            true));
+
+    // Test 1 truncate up
+    Assert.assertEquals(
+        "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+        truncateBytesAsHex(
+            Hex.decodeHex(
+                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
+            false));
+
+    Assert.assertEquals(
+        "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab",
+        truncateBytesAsHex(
+            Hex.decodeHex(
+                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
+            true));
+
+    // Test one overflow
+    Assert.assertEquals(
+        "aaaaaaaaaaaaaaaaaaaaaaaaaaaaafffffffffffffffffffffffffffffffaaff",
+        truncateBytesAsHex(
+            Hex.decodeHex("aaaaaaaaaaaaaaaaaaaaaaaaaaaaafffffffffffffffffffffffffffffffaaffffffff"),
+            false));
+    Assert.assertEquals(
+        "aaaaaaaaaaaaaaaaaaaaaaaaaaaaafffffffffffffffffffffffffffffffab00",
+        truncateBytesAsHex(
+            Hex.decodeHex("aaaaaaaaaaaaaaaaaaaaaaaaaaaaafffffffffffffffffffffffffffffffaaffffffff"),
+            true));
+
+    // Test many overflow
+    Assert.assertEquals(
+        "aaaaaaaaaaaaaaaaaaaaaaaaaaaaafffffffffffffffffffffffffffffffffff",
+        truncateBytesAsHex(
+            Hex.decodeHex(
+                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaafffffffffffffffffffffffffffffffffffffffffffffffffffff"),
+            false));
+    Assert.assertEquals(
+        "aaaaaaaaaaaaaaaaaaaaaaaaaaaab00000000000000000000000000000000000",
+        truncateBytesAsHex(
+            Hex.decodeHex(
+                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaafffffffffffffffffffffffffffffffffffffffffffffffffffff"),
+            true));
+
+    // Test infinity
+    Assert.assertEquals(
+        "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+        truncateBytesAsHex(
+            Hex.decodeHex(
+                "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffcccccccccccc"),
+            false));
+    Assert.assertEquals(
+        "Z",
+        truncateBytesAsHex(
+            Hex.decodeHex(
+                "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffcccccccccccc"),
+            true));
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelDataTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelDataTest.java
index f429c0d84..5b545d697 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelDataTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelDataTest.java
@@ -1,6 +1,7 @@
 package net.snowflake.ingest.streaming.internal;
 
 import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
 import net.snowflake.ingest.utils.ErrorCode;
@@ -107,13 +108,15 @@ public void testGetCombinedColumnStatsMap() {
     Assert.assertEquals(new BigInteger("10"), oneCombined.getCurrentMinIntValue());
     Assert.assertEquals(new BigInteger("17"), oneCombined.getCurrentMaxIntValue());
     Assert.assertEquals(-1, oneCombined.getDistinctValues());
-    Assert.assertNull(oneCombined.getCurrentMinColStrValue());
-    Assert.assertNull(oneCombined.getCurrentMaxColStrValue());
+    Assert.assertNull(oneCombined.getCurrentMinStrValue());
+    Assert.assertNull(oneCombined.getCurrentMaxStrValue());
     Assert.assertNull(oneCombined.getCurrentMinRealValue());
     Assert.assertNull(oneCombined.getCurrentMaxRealValue());
 
-    Assert.assertEquals("10", twoCombined.getCurrentMinColStrValue());
-    Assert.assertEquals("17", twoCombined.getCurrentMaxColStrValue());
+    Assert.assertArrayEquals(
+        "10".getBytes(StandardCharsets.UTF_8), twoCombined.getCurrentMinStrValue());
+    Assert.assertArrayEquals(
+        "17".getBytes(StandardCharsets.UTF_8), twoCombined.getCurrentMaxStrValue());
     Assert.assertEquals(-1, twoCombined.getDistinctValues());
     Assert.assertNull(twoCombined.getCurrentMinIntValue());
     Assert.assertNull(twoCombined.getCurrentMaxIntValue());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index bb5da4865..eff9788cf 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -1,5 +1,6 @@
 package net.snowflake.ingest.streaming.internal;
 
+import static net.snowflake.ingest.TestUtils.buildString;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.BYTES_16_MB;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.BYTES_8_MB;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.isAllowedSemiStructuredType;
@@ -364,32 +365,39 @@ public void testValidateAndParseBigDecimal() {
   public void testValidateAndParseString() {
     assertEquals("honk", validateAndParseString("COL", "honk", Optional.empty()));
 
-    // Check max String length
-    StringBuilder longBuilder = new StringBuilder();
-    for (int i = 0; i < BYTES_16_MB; i++) {
-      longBuilder.append("č"); // max string length is measured in chars, not bytes
-    }
-    String maxString = longBuilder.toString();
+    // Check max byte length
+    String maxString = buildString("a", BYTES_16_MB);
     assertEquals(maxString, validateAndParseString("COL", maxString, Optional.empty()));
 
-    // max length - 1 should also succeed
-    longBuilder.setLength(BYTES_16_MB - 1);
-    String maxStringMinusOne = longBuilder.toString();
+    // max byte length - 1 should also succeed
+    String maxStringMinusOne = buildString("a", BYTES_16_MB - 1);
     assertEquals(
         maxStringMinusOne, validateAndParseString("COL", maxStringMinusOne, Optional.empty()));
 
-    // max length + 1 should fail
+    // max byte length + 1 should fail
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseString("COL", longBuilder.append("aa").toString(), Optional.empty()));
+        () -> validateAndParseString("COL", maxString + "a", Optional.empty()));
+
+    // Test that max character length validation counts characters and not bytes
+    assertEquals("a", validateAndParseString("COL", "a", Optional.of(1)));
+    assertEquals("č", validateAndParseString("COL", "č", Optional.of(1)));
+    assertEquals("❄", validateAndParseString("COL", "❄", Optional.of(1)));
+    assertEquals("🍞", validateAndParseString("COL", "🍞", Optional.of(1)));
 
-    // Test max length validation
+    // Test max character length rejection
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", "a🍞", Optional.of(1)));
     expectError(
         ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", "12345", Optional.of(4)));
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", false, Optional.of(4)));
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", 12345, Optional.of(4)));
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", 1.2345, Optional.of(4)));
 
+    // Test that invalid UTF-8 strings cannot be ingested
+    expectError(
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseString("COL", "foo\uD800bar", Optional.empty()));
+
     // Test unsupported values
     expectError(
         ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", new Object(), Optional.empty()));
@@ -433,6 +441,9 @@ public void testValidateAndParseVariant() throws Exception {
     assertNull(validateAndParseVariant("COL", ""));
     assertNull(validateAndParseVariant("COL", "  "));
 
+    // Test that invalid UTF-8 strings cannot be ingested
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", "\"foo\uD800bar\""));
+
     // Test forbidden values
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", "{null}"));
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", "}{"));
@@ -491,6 +502,9 @@ public void testValidateAndParseArray() throws Exception {
     assertEquals("[null]", validateAndParseArray("COL", "null"));
     assertEquals("[null]", validateAndParseArray("COL", null));
 
+    // Test that invalid UTF-8 strings cannot be ingested
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseArray("COL", "\"foo\uD800bar\""));
+
     // Test forbidden values
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseArray("COL", readTree("[]")));
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseArray("COL", new Object()));
@@ -538,6 +552,10 @@ public void testValidateAndParseObject() throws Exception {
       assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), err.getVendorCode());
     }
 
+    // Test that invalid UTF-8 strings cannot be ingested
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", "{\"foo\": \"foo\uD800bar\"}"));
+
     // Test forbidden values
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", readTree("{}")));
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", "[]"));
@@ -588,8 +606,8 @@ public void testTooLargeMultiByteSemiStructuredValues() {
     m.put("a", new String(stringContent));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: {a=ČČČČČČČČČČČČČČČČČ.... Value cannot"
-            + " be ingested into Snowflake column COL of type VARIANT: Variant too long:"
+        "The given row cannot be converted to Arrow format: {\"a\":\"ČČČČČČČČČČČČČČ.... Value"
+            + " cannot be ingested into Snowflake column COL of type VARIANT: Variant too long:"
             + " length=18874376 maxLength=16777152",
         () -> validateAndParseVariant("COL", m));
     expectErrorCodeAndMessage(
@@ -1033,7 +1051,8 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column COL of type STRING: String too long: length=3 maxLength=2",
+            + " Snowflake column COL of type STRING: String too long: length=3 characters"
+            + " maxLength=2 characters",
         () -> validateAndParseString("COL", "abc", Optional.of(2)));
 
     // BINARY
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FileColumnPropertiesTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FileColumnPropertiesTest.java
new file mode 100644
index 000000000..e056f4ab6
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FileColumnPropertiesTest.java
@@ -0,0 +1,30 @@
+package net.snowflake.ingest.streaming.internal;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class FileColumnPropertiesTest {
+
+  @Test
+  public void testFileColumnPropertiesConstructor() {
+    // Test simple construction
+    RowBufferStats stats = new RowBufferStats("COL");
+    stats.addStrValue("bcd");
+    stats.addStrValue("abcde");
+    FileColumnProperties props = new FileColumnProperties(stats);
+    Assert.assertEquals("6162636465", props.getMinStrValue());
+    Assert.assertNull(props.getMinStrNonCollated());
+    Assert.assertEquals("626364", props.getMaxStrValue());
+    Assert.assertNull(props.getMaxStrNonCollated());
+
+    // Test that truncation is performed
+    stats = new RowBufferStats("COL");
+    stats.addStrValue("aßßßßßßßßßßßßßßßß");
+    Assert.assertEquals(33, stats.getCurrentMinStrValue().length);
+    props = new FileColumnProperties(stats);
+    Assert.assertNull(props.getMinStrNonCollated());
+    Assert.assertNull(props.getMaxStrNonCollated());
+    Assert.assertEquals(32 * 2, props.getMinStrValue().length());
+    Assert.assertEquals(32 * 2, props.getMaxStrValue().length());
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 884b7fde0..b4554f481 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -388,7 +388,6 @@ private static ColumnMetadata createTestTextColumn() {
     colChar.setByteLength(14);
     colChar.setLength(11);
     colChar.setScale(0);
-    colChar.setCollation("en-ci");
     return colChar;
   }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
index 459b4f5cd..cc0361147 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
@@ -2,6 +2,7 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
 import net.snowflake.ingest.utils.SFException;
@@ -243,7 +244,7 @@ public void parseValueBinary() {
         .expectedValueClass(byte[].class)
         .expectedParsedValue("1234abcd".getBytes())
         .expectedSize(8.0f)
-        .expectedMinMax("1234abcd")
+        .expectedMinMax("1234abcd".getBytes(StandardCharsets.UTF_8))
         .assertMatches();
   }
 
@@ -598,17 +599,22 @@ void assertMatches() {
         Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinIntValue());
         Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxIntValue());
         return;
-      } else if (minMaxStat instanceof String || valueClass.equals(String.class)) {
+      } else if (minMaxStat instanceof byte[]) {
+        Assert.assertArrayEquals((byte[]) minMaxStat, rowBufferStats.getCurrentMinStrValue());
+        Assert.assertArrayEquals((byte[]) minMaxStat, rowBufferStats.getCurrentMaxStrValue());
+        return;
+      } else if (valueClass.equals(String.class)) {
         // String can have null min/max stats for variant data types
-        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinColStrValue());
-        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxColStrValue());
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinStrValue());
+        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxStrValue());
         return;
       } else if (minMaxStat instanceof Double || minMaxStat instanceof BigDecimal) {
         Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinRealValue());
         Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxRealValue());
         return;
       }
-      throw new IllegalArgumentException("Unknown data type for min stat");
+      throw new IllegalArgumentException(
+          String.format("Unknown data type for min stat: %s", minMaxStat.getClass()));
     }
 
     void assertNull() {
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferStatsTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferStatsTest.java
index fa3a38c07..9f2e848f3 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferStatsTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferStatsTest.java
@@ -1,72 +1,12 @@
 package net.snowflake.ingest.streaming.internal;
 
 import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
 import org.junit.Assert;
 import org.junit.Test;
 
 public class RowBufferStatsTest {
 
-  @Test
-  public void testCollationStates() throws Exception {
-    RowBufferStats ai = new RowBufferStats("COL1", "en-ai");
-    RowBufferStats as = new RowBufferStats("COL1", "en-as");
-    RowBufferStats pi = new RowBufferStats("COL1", "en-pi");
-    RowBufferStats ps = new RowBufferStats("COL1", "en-ps");
-    RowBufferStats fu = new RowBufferStats("COL1", "en-fu");
-    RowBufferStats fl = new RowBufferStats("COL1", "en-fl");
-    RowBufferStats lower = new RowBufferStats("COL1", "lower");
-    RowBufferStats upper = new RowBufferStats("COL1", "upper");
-    RowBufferStats ltrim = new RowBufferStats("COL1", "ltrim");
-    RowBufferStats rtrim = new RowBufferStats("COL1", "rtrim");
-    RowBufferStats trim = new RowBufferStats("COL1", "trim");
-
-    // Accents
-    ai.addStrValue("a");
-    ai.addStrValue("à");
-    as.addStrValue("a");
-    as.addStrValue("à");
-    Assert.assertEquals("a", ai.getCurrentMinColStrValue());
-    Assert.assertEquals("a", ai.getCurrentMaxColStrValue());
-    Assert.assertEquals("a", as.getCurrentMinColStrValue());
-    Assert.assertEquals("à", as.getCurrentMaxColStrValue());
-
-    // Punctuation
-    pi.addStrValue(".b");
-    pi.addStrValue("a");
-    ps.addStrValue(".b");
-    ps.addStrValue("a");
-
-    Assert.assertEquals("a", pi.getCurrentMinColStrValue());
-    Assert.assertEquals(".b", ps.getCurrentMinColStrValue());
-
-    // First Lower and Upper
-    fl.addStrValue("C");
-    fl.addStrValue("b");
-    fu.addStrValue("b");
-    fu.addStrValue("C");
-    Assert.assertEquals("b", fl.getCurrentMinColStrValue());
-    Assert.assertEquals("b", fu.getCurrentMinColStrValue());
-
-    // Lower and Upper
-    lower.addStrValue("AA");
-    lower.addStrValue("a");
-    upper.addStrValue("AA");
-    upper.addStrValue("aaa");
-    Assert.assertEquals("a", lower.getCurrentMinColStrValue());
-    Assert.assertEquals("AA", upper.getCurrentMinColStrValue());
-
-    // Trim settings
-    trim.addStrValue(" z ");
-    trim.addStrValue("b");
-    ltrim.addStrValue(" z");
-    ltrim.addStrValue("b");
-    rtrim.addStrValue("z ");
-    rtrim.addStrValue("b");
-    Assert.assertEquals("b", trim.getCurrentMinColStrValue());
-    Assert.assertEquals("b", ltrim.getCurrentMinColStrValue());
-    Assert.assertEquals("b", rtrim.getCurrentMinColStrValue());
-  }
-
   @Test
   public void testEmptyState() throws Exception {
     RowBufferStats stats = new RowBufferStats("COL1");
@@ -74,8 +14,8 @@ public void testEmptyState() throws Exception {
     Assert.assertNull(stats.getCollationDefinitionString());
     Assert.assertNull(stats.getCurrentMinRealValue());
     Assert.assertNull(stats.getCurrentMaxRealValue());
-    Assert.assertNull(stats.getCurrentMinColStrValue());
-    Assert.assertNull(stats.getCurrentMaxColStrValue());
+    Assert.assertNull(stats.getCurrentMinStrValue());
+    Assert.assertNull(stats.getCurrentMaxStrValue());
     Assert.assertNull(stats.getCurrentMinIntValue());
     Assert.assertNull(stats.getCurrentMaxIntValue());
 
@@ -88,18 +28,21 @@ public void testMinMaxStrNonCol() throws Exception {
     RowBufferStats stats = new RowBufferStats("COL1");
 
     stats.addStrValue("bob");
-    Assert.assertEquals("bob", stats.getCurrentMinColStrValue());
-    Assert.assertEquals("bob", stats.getCurrentMaxColStrValue());
+    Assert.assertArrayEquals("bob".getBytes(StandardCharsets.UTF_8), stats.getCurrentMinStrValue());
+    Assert.assertArrayEquals("bob".getBytes(StandardCharsets.UTF_8), stats.getCurrentMaxStrValue());
     Assert.assertEquals(-1, stats.getDistinctValues());
 
     stats.addStrValue("charlie");
-    Assert.assertEquals("bob", stats.getCurrentMinColStrValue());
-    Assert.assertEquals("charlie", stats.getCurrentMaxColStrValue());
+    Assert.assertArrayEquals("bob".getBytes(StandardCharsets.UTF_8), stats.getCurrentMinStrValue());
+    Assert.assertArrayEquals(
+        "charlie".getBytes(StandardCharsets.UTF_8), stats.getCurrentMaxStrValue());
     Assert.assertEquals(-1, stats.getDistinctValues());
 
     stats.addStrValue("alice");
-    Assert.assertEquals("alice", stats.getCurrentMinColStrValue());
-    Assert.assertEquals("charlie", stats.getCurrentMaxColStrValue());
+    Assert.assertArrayEquals(
+        "alice".getBytes(StandardCharsets.UTF_8), stats.getCurrentMinStrValue());
+    Assert.assertArrayEquals(
+        "charlie".getBytes(StandardCharsets.UTF_8), stats.getCurrentMaxStrValue());
     Assert.assertEquals(-1, stats.getDistinctValues());
 
     Assert.assertNull(stats.getCurrentMinRealValue());
@@ -110,55 +53,6 @@ public void testMinMaxStrNonCol() throws Exception {
     Assert.assertEquals(0, stats.getCurrentNullCount());
   }
 
-  @Test
-  public void testStrTruncation() throws Exception {
-    RowBufferStats stats = new RowBufferStats("COL1");
-    stats.addStrValue("abcde|abcde|abcde|abcde|abcde|abcde|");
-    Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ab", stats.getCurrentMinColStrValue());
-    Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ac", stats.getCurrentMaxColStrValue());
-
-    stats.addStrValue("zabcde|abcde|abcde|abcde|abcde|abcde|");
-    Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ab", stats.getCurrentMinColStrValue());
-    Assert.assertEquals("zabcde|abcde|abcde|abcde|abcde|b", stats.getCurrentMaxColStrValue());
-
-    RowBufferStats ai = new RowBufferStats("COL1", "en-ai");
-    ai.addStrValue("abcde|abcde|abcde|abcde|abcde|abcde|");
-    Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ab", ai.getCurrentMinColStrValue());
-    Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ac", ai.getCurrentMaxColStrValue());
-
-    ai.addStrValue("zabcde|abcde|abcde|abcde|abcde|abcde|");
-    Assert.assertEquals("abcde|abcde|abcde|abcde|abcde|ab", ai.getCurrentMinColStrValue());
-    Assert.assertEquals("zabcde|abcde|abcde|abcde|abcde|b", ai.getCurrentMaxColStrValue());
-  }
-
-  @Test
-  public void testMinMaxStrCol() throws Exception {
-    RowBufferStats stats = new RowBufferStats("COL1", "en-ci");
-
-    Assert.assertEquals("en-ci", stats.getCollationDefinitionString());
-
-    stats.addStrValue("bob");
-    Assert.assertEquals("bob", stats.getCurrentMinColStrValue());
-    Assert.assertEquals("bob", stats.getCurrentMaxColStrValue());
-    Assert.assertEquals(-1, stats.getDistinctValues());
-
-    stats.addStrValue("Bob");
-    Assert.assertEquals("bob", stats.getCurrentMinColStrValue());
-    Assert.assertEquals("bob", stats.getCurrentMaxColStrValue());
-    Assert.assertEquals(-1, stats.getDistinctValues());
-
-    stats.addStrValue("Alice");
-    Assert.assertEquals("Alice", stats.getCurrentMinColStrValue());
-    Assert.assertEquals("bob", stats.getCurrentMaxColStrValue());
-    Assert.assertEquals(-1, stats.getDistinctValues());
-
-    Assert.assertNull(stats.getCurrentMinRealValue());
-    Assert.assertNull(stats.getCurrentMaxRealValue());
-    Assert.assertNull(stats.getCurrentMinIntValue());
-    Assert.assertNull(stats.getCurrentMaxIntValue());
-    Assert.assertEquals(0, stats.getCurrentNullCount());
-  }
-
   @Test
   public void testMinMaxInt() throws Exception {
     RowBufferStats stats = new RowBufferStats("COL1");
@@ -181,8 +75,8 @@ public void testMinMaxInt() throws Exception {
     Assert.assertNull(stats.getCurrentMinRealValue());
     Assert.assertNull(stats.getCurrentMaxRealValue());
     Assert.assertNull(stats.getCollationDefinitionString());
-    Assert.assertNull(stats.getCurrentMinColStrValue());
-    Assert.assertNull(stats.getCurrentMaxColStrValue());
+    Assert.assertNull(stats.getCurrentMinStrValue());
+    Assert.assertNull(stats.getCurrentMaxStrValue());
     Assert.assertEquals(0, stats.getCurrentNullCount());
   }
 
@@ -208,8 +102,8 @@ public void testMinMaxReal() throws Exception {
     Assert.assertNull(stats.getCurrentMinIntValue());
     Assert.assertNull(stats.getCurrentMaxIntValue());
     Assert.assertNull(stats.getCollationDefinitionString());
-    Assert.assertNull(stats.getCurrentMinColStrValue());
-    Assert.assertNull(stats.getCurrentMaxColStrValue());
+    Assert.assertNull(stats.getCurrentMinStrValue());
+    Assert.assertNull(stats.getCurrentMaxStrValue());
     Assert.assertEquals(0, stats.getCurrentNullCount());
   }
 
@@ -259,8 +153,8 @@ public void testGetCombinedStats() throws Exception {
     Assert.assertEquals(-1, result.getDistinctValues());
     Assert.assertEquals(2, result.getCurrentNullCount());
 
-    Assert.assertNull(result.getCurrentMinColStrValue());
-    Assert.assertNull(result.getCurrentMaxColStrValue());
+    Assert.assertNull(result.getCurrentMinStrValue());
+    Assert.assertNull(result.getCurrentMaxStrValue());
     Assert.assertNull(result.getCurrentMinRealValue());
     Assert.assertNull(result.getCurrentMaxRealValue());
 
@@ -285,8 +179,8 @@ public void testGetCombinedStats() throws Exception {
     Assert.assertEquals(0, result.getCurrentNullCount());
 
     Assert.assertNull(result.getCollationDefinitionString());
-    Assert.assertNull(result.getCurrentMinColStrValue());
-    Assert.assertNull(result.getCurrentMaxColStrValue());
+    Assert.assertNull(result.getCurrentMinStrValue());
+    Assert.assertNull(result.getCurrentMaxStrValue());
     Assert.assertNull(result.getCurrentMinIntValue());
     Assert.assertNull(result.getCurrentMaxIntValue());
 
@@ -309,8 +203,8 @@ public void testGetCombinedStats() throws Exception {
     two.setCurrentMaxLength(1);
 
     result = RowBufferStats.getCombinedStats(one, two);
-    Assert.assertEquals("a", result.getCurrentMinColStrValue());
-    Assert.assertEquals("g", result.getCurrentMaxColStrValue());
+    Assert.assertArrayEquals("a".getBytes(StandardCharsets.UTF_8), result.getCurrentMinStrValue());
+    Assert.assertArrayEquals("g".getBytes(StandardCharsets.UTF_8), result.getCurrentMaxStrValue());
     Assert.assertEquals(-1, result.getDistinctValues());
     Assert.assertEquals(2, result.getCurrentNullCount());
     Assert.assertEquals(5, result.getCurrentMaxLength());
@@ -319,33 +213,6 @@ public void testGetCombinedStats() throws Exception {
     Assert.assertNull(result.getCurrentMaxRealValue());
     Assert.assertNull(result.getCurrentMinIntValue());
     Assert.assertNull(result.getCurrentMaxIntValue());
-
-    // Test for Strings with collation
-    one = new RowBufferStats("COL1", "en-ci");
-    two = new RowBufferStats("COL1", "en-ci");
-
-    one.addStrValue("a");
-    one.addStrValue("d");
-    one.addStrValue("f");
-    one.addStrValue("g");
-    one.incCurrentNullCount();
-
-    two.addStrValue("Alpha");
-    two.addStrValue("b");
-    two.addStrValue("c");
-    two.addStrValue("d");
-    two.incCurrentNullCount();
-
-    result = RowBufferStats.getCombinedStats(one, two);
-    Assert.assertEquals("a", result.getCurrentMinColStrValue());
-    Assert.assertEquals("g", result.getCurrentMaxColStrValue());
-    Assert.assertEquals(-1, result.getDistinctValues());
-    Assert.assertEquals(2, result.getCurrentNullCount());
-
-    Assert.assertNull(result.getCurrentMinRealValue());
-    Assert.assertNull(result.getCurrentMaxRealValue());
-    Assert.assertNull(result.getCurrentMinIntValue());
-    Assert.assertNull(result.getCurrentMaxIntValue());
   }
 
   @Test
@@ -367,8 +234,8 @@ public void testGetCombinedStatsNull() throws Exception {
     Assert.assertEquals(-1, result.getDistinctValues());
     Assert.assertEquals(2, result.getCurrentNullCount());
 
-    Assert.assertNull(result.getCurrentMinColStrValue());
-    Assert.assertNull(result.getCurrentMaxColStrValue());
+    Assert.assertNull(result.getCurrentMinStrValue());
+    Assert.assertNull(result.getCurrentMaxStrValue());
     Assert.assertNull(result.getCurrentMinRealValue());
     Assert.assertNull(result.getCurrentMaxRealValue());
 
@@ -386,8 +253,8 @@ public void testGetCombinedStatsNull() throws Exception {
     Assert.assertEquals(-1, result.getDistinctValues());
     Assert.assertEquals(0, result.getCurrentNullCount());
 
-    Assert.assertNull(result.getCurrentMinColStrValue());
-    Assert.assertNull(result.getCurrentMaxColStrValue());
+    Assert.assertNull(result.getCurrentMinStrValue());
+    Assert.assertNull(result.getCurrentMaxStrValue());
     Assert.assertNull(result.getCurrentMinIntValue());
     Assert.assertNull(result.getCurrentMaxIntValue());
 
@@ -402,8 +269,9 @@ public void testGetCombinedStatsNull() throws Exception {
     one.incCurrentNullCount();
 
     result = RowBufferStats.getCombinedStats(one, two);
-    Assert.assertEquals("alpha", result.getCurrentMinColStrValue());
-    Assert.assertEquals("g", result.getCurrentMaxColStrValue());
+    Assert.assertArrayEquals(
+        "alpha".getBytes(StandardCharsets.UTF_8), result.getCurrentMinStrValue());
+    Assert.assertArrayEquals("g".getBytes(StandardCharsets.UTF_8), result.getCurrentMaxStrValue());
     Assert.assertEquals(-1, result.getDistinctValues());
     Assert.assertEquals(1, result.getCurrentNullCount());
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 3538d7d7b..b867acf95 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -16,6 +16,7 @@
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.commons.codec.binary.Hex;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -109,7 +110,6 @@ static List<ColumnMetadata> createSchema() {
     colChar.setByteLength(14);
     colChar.setLength(11);
     colChar.setScale(0);
-    colChar.setCollation("en-ci");
 
     return Arrays.asList(
         colTinyIntCase, colTinyInt, colSmallInt, colInt, colBigInt, colDecimal, colChar);
@@ -128,6 +128,27 @@ private AbstractRowBuffer<?> createTestBuffer(OpenChannelRequest.OnErrorOption o
         enableParquetMemoryOptimization);
   }
 
+  @Test
+  public void testCollatedColumnsAreRejected() {
+    AbstractRowBuffer<?> testBuffer = createTestBuffer(OpenChannelRequest.OnErrorOption.ABORT);
+
+    ColumnMetadata collatedColumn = new ColumnMetadata();
+    collatedColumn.setName("COLCHAR");
+    collatedColumn.setPhysicalType("LOB");
+    collatedColumn.setNullable(true);
+    collatedColumn.setLogicalType("TEXT");
+    collatedColumn.setByteLength(14);
+    collatedColumn.setLength(11);
+    collatedColumn.setScale(0);
+    collatedColumn.setCollation("en-ci");
+    try {
+      this.rowBufferOnErrorAbort.setupSchema(Collections.singletonList(collatedColumn));
+      Assert.fail("Collated columns are not supported");
+    } catch (SFException e) {
+      Assert.assertEquals(ErrorCode.UNSUPPORTED_DATA_TYPE.getMessageCode(), e.getVendorCode());
+    }
+  }
+
   @Test
   public void buildFieldErrorStates() {
     // Nonsense Type
@@ -201,12 +222,12 @@ public void testReset() {
     RowBufferStats stats = this.rowBufferOnErrorContinue.statsMap.get("COLCHAR");
     stats.addIntValue(BigInteger.valueOf(1));
     Assert.assertEquals(BigInteger.valueOf(1), stats.getCurrentMaxIntValue());
-    Assert.assertEquals("en-ci", stats.getCollationDefinitionString());
+    Assert.assertNull(stats.getCollationDefinitionString());
     this.rowBufferOnErrorContinue.reset();
     RowBufferStats resetStats = this.rowBufferOnErrorContinue.statsMap.get("COLCHAR");
     Assert.assertNotNull(resetStats);
     Assert.assertNull(resetStats.getCurrentMaxIntValue());
-    Assert.assertEquals("en-ci", resetStats.getCollationDefinitionString());
+    Assert.assertNull(resetStats.getCollationDefinitionString());
   }
 
   @Test
@@ -501,8 +522,12 @@ public void testBuildEpInfoFromStats() {
 
     FileColumnProperties strColumnResult = columnResults.get("strColumn");
     Assert.assertEquals(-1, strColumnResult.getDistinctValues());
-    Assert.assertEquals("alice", strColumnResult.getMinStrValue());
-    Assert.assertEquals("bob", strColumnResult.getMaxStrValue());
+    Assert.assertEquals(
+        Hex.encodeHexString("alice".getBytes(StandardCharsets.UTF_8)),
+        strColumnResult.getMinStrValue());
+    Assert.assertEquals(
+        Hex.encodeHexString("bob".getBytes(StandardCharsets.UTF_8)),
+        strColumnResult.getMaxStrValue());
     Assert.assertEquals(1, strColumnResult.getNullCount());
 
     FileColumnProperties intColumnResult = columnResults.get("intColumn");
@@ -678,8 +703,11 @@ private void testStatsE2EHelper(AbstractRowBuffer<?> rowBuffer) {
     Assert.assertEquals(0, columnEpStats.get("COLBIGINT").getCurrentNullCount());
     Assert.assertEquals(-1, columnEpStats.get("COLBIGINT").getDistinctValues());
 
-    Assert.assertEquals("alice", columnEpStats.get("COLCHAR").getCurrentMaxColStrValue());
-    Assert.assertEquals("2", columnEpStats.get("COLCHAR").getCurrentMinColStrValue());
+    Assert.assertArrayEquals(
+        "2".getBytes(StandardCharsets.UTF_8), columnEpStats.get("COLCHAR").getCurrentMinStrValue());
+    Assert.assertArrayEquals(
+        "alice".getBytes(StandardCharsets.UTF_8),
+        columnEpStats.get("COLCHAR").getCurrentMaxStrValue());
     Assert.assertEquals(0, columnEpStats.get("COLCHAR").getCurrentNullCount());
     Assert.assertEquals(-1, columnEpStats.get("COLCHAR").getDistinctValues());
 
@@ -1102,10 +1130,12 @@ private void testE2EBinaryHelper(OpenChannelRequest.OnErrorOption onErrorOption)
 
     Assert.assertEquals(3, result.getRowCount());
     Assert.assertEquals(11L, result.getColumnEps().get("COLBINARY").getCurrentMaxLength());
-    Assert.assertEquals(
-        "Hello World", result.getColumnEps().get("COLBINARY").getCurrentMinColStrValue());
-    Assert.assertEquals(
-        "Honk Honk", result.getColumnEps().get("COLBINARY").getCurrentMaxColStrValue());
+    Assert.assertArrayEquals(
+        "Hello World".getBytes(StandardCharsets.UTF_8),
+        result.getColumnEps().get("COLBINARY").getCurrentMinStrValue());
+    Assert.assertArrayEquals(
+        "Honk Honk".getBytes(StandardCharsets.UTF_8),
+        result.getColumnEps().get("COLBINARY").getCurrentMaxStrValue());
     Assert.assertEquals(1, result.getColumnEps().get("COLBINARY").getCurrentNullCount());
   }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index b8ceee912..cf3f6f76a 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -287,57 +287,6 @@ public void testMultiChannelChunk() {
                 INTERLEAVED_CHANNEL_TABLE + "_channel_" + i));
   }
 
-  @Test
-  public void testCollation() throws Exception {
-    String collationTable = "collation_table";
-    jdbcConnection
-        .createStatement()
-        .execute(
-            String.format(
-                "create or replace table %s (noncol char(10), col char(10) collate 'en-ci');",
-                collationTable));
-
-    OpenChannelRequest request1 =
-        OpenChannelRequest.builder("CHANNEL")
-            .setDBName(testDb)
-            .setSchemaName(TEST_SCHEMA)
-            .setTableName(collationTable)
-            .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
-            .build();
-
-    // Open a streaming ingest channel from the given client
-    SnowflakeStreamingIngestChannel channel1 = client.openChannel(request1);
-    Map<String, Object> row = new HashMap<>();
-    row.put("col", "AA");
-    row.put("noncol", "AA");
-    verifyInsertValidationResponse(channel1.insertRow(row, "1"));
-    row.put("col", "a");
-    row.put("noncol", "a");
-    verifyInsertValidationResponse(channel1.insertRow(row, "2"));
-
-    // Close the channel after insertion
-    channel1.close().get();
-
-    for (int i = 1; i < 15; i++) {
-      if (channel1.getLatestCommittedOffsetToken() != null
-          && channel1.getLatestCommittedOffsetToken().equals("2")) {
-        ResultSet result =
-            jdbcConnection
-                .createStatement()
-                .executeQuery(
-                    String.format(
-                        "select min(col), min(noncol) from %s.%s.%s",
-                        testDb, TEST_SCHEMA, collationTable));
-        result.next();
-        Assert.assertEquals("a", result.getString(1));
-        Assert.assertEquals("AA", result.getString(2));
-        return;
-      }
-      Thread.sleep(1000);
-    }
-    Assert.fail("Row sequencer not updated before timeout");
-  }
-
   @Test
   public void testDecimalColumnIngest() throws Exception {
     String decimalTableName = "decimal_table";
@@ -1313,14 +1262,14 @@ private void assertNullRow(Map<String, Object> expectedNullRow, ResultSet actual
       throws SQLException {
     Assert.assertNotNull(actualResult);
     assertNonTimeAndVarFields(expectedNullRow, actualResult);
-    Assert.assertEquals(null, actualResult.getString("VAR"));
-    Assert.assertEquals(null, actualResult.getString("OBJ"));
-    Assert.assertEquals(null, actualResult.getString("ARR"));
-    Assert.assertEquals(null, actualResult.getDate("EPOCHDAYS"));
-    Assert.assertEquals(null, actualResult.getTimestamp("EPOCHSEC"));
-    Assert.assertEquals(null, actualResult.getTimestamp("EPOCHNANO"));
-    Assert.assertEquals(null, actualResult.getTimestamp("TIMESEC"));
-    Assert.assertEquals(null, actualResult.getTimestamp("TIMENANO"));
+    Assert.assertNull(actualResult.getString("VAR"));
+    Assert.assertNull(actualResult.getString("OBJ"));
+    Assert.assertNull(actualResult.getString("ARR"));
+    Assert.assertNull(actualResult.getDate("EPOCHDAYS"));
+    Assert.assertNull(actualResult.getTimestamp("EPOCHSEC"));
+    Assert.assertNull(actualResult.getTimestamp("EPOCHNANO"));
+    Assert.assertNull(actualResult.getTimestamp("TIMESEC"));
+    Assert.assertNull(actualResult.getTimestamp("TIMENANO"));
   }
 
   private void assertPosRow(Map<String, Object> expectedPosRow, ResultSet actualResult)
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index f6de25e31..d18e735ba 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -55,11 +55,6 @@ public static Collection<Object[]> bdecVersion() {
 
   private String schemaName = "PUBLIC";
   private SnowflakeStreamingIngestClient client;
-
-  protected String randomString() {
-    return UUID.randomUUID().toString().replace("-", "_");
-  }
-
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
   private final Constants.BdecVersion bdecVersion;
@@ -71,7 +66,7 @@ public AbstractDataTypeTest(
 
   @Before
   public void before() throws Exception {
-    databaseName = String.format("SDK_DATATYPE_COMPATIBILITY_IT_%s", randomString());
+    databaseName = String.format("SDK_DATATYPE_COMPATIBILITY_IT_%s", getRandomIdentifier());
     conn = TestUtils.getConnection(true);
     conn.createStatement().execute(String.format("create or replace database %s;", databaseName));
     conn.createStatement().execute(String.format("use database %s;", databaseName));
@@ -107,15 +102,7 @@ public void after() throws Exception {
   }
 
   protected String createTable(String dataType) throws SQLException {
-    String tableName =
-        String.format("test_%s_%s", dataType, UUID.randomUUID())
-            .replace('-', '_')
-            .replace(' ', '_')
-            .replace('(', '_')
-            .replace(')', '_')
-            .replace(',', '_');
-
-    //    System.out.printf("Creating table %s.%s.%s%n", databaseName, schemaName, tableName);
+    String tableName = getRandomIdentifier();
     conn.createStatement()
         .execute(
             String.format(
@@ -124,6 +111,10 @@ protected String createTable(String dataType) throws SQLException {
     return tableName;
   }
 
+  protected String getRandomIdentifier() {
+    return String.format("test_%s", UUID.randomUUID()).replace('-', '_');
+  }
+
   protected SnowflakeStreamingIngestChannel openChannel(String tableName) {
     return openChannel(tableName, OpenChannelRequest.OnErrorOption.ABORT);
   }
@@ -140,7 +131,7 @@ protected SnowflakeStreamingIngestChannel openChannel(
     return client.openChannel(openChannelRequest);
   }
 
-  private Map<String, Object> createStreamingIngestRow(Object value) {
+  protected Map<String, Object> createStreamingIngestRow(Object value) {
     Map<String, Object> row = new HashMap<>();
     row.put(SOURCE_COLUMN_NAME, SOURCE_STREAMING_INGEST);
     row.put(VALUE_COLUMN_NAME, value);
@@ -193,6 +184,15 @@ protected <T> void expectArrowNotSupported(String dataType, T value) throws Exce
                 && x.getMessage().contains("The given row cannot be converted to Arrow format")));
   }
 
+  /**
+   * Simplified version, which does not insert using JDBC. Useful for testing non-JDBC types like
+   * BigInteger, java.time.* types, etc.
+   */
+  <VALUE> void testIngestion(String dataType, VALUE expectedValue, Provider<VALUE> selectProvider)
+      throws Exception {
+    ingestAndAssert(dataType, expectedValue, null, expectedValue, null, selectProvider);
+  }
+
   /**
    * Simplified version, which does not insert using JDBC. Useful for testing non-JDBC types like
    * BigInteger, java.time.* types, etc.
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
index 95681863f..4f5ac6e78 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
@@ -1,13 +1,24 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
+import static net.snowflake.ingest.TestUtils.buildString;
+
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.sql.SQLException;
+import net.snowflake.ingest.TestUtils;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.SFException;
+import org.junit.Assert;
 import org.junit.Ignore;
 import org.junit.Test;
 
 public class StringsIT extends AbstractDataTypeTest {
 
+  private static final int MB_16 = 16 * 1024 * 1024;
+
   public StringsIT(String name, Constants.BdecVersion bdecVersion) {
     super(name, bdecVersion);
   }
@@ -18,9 +29,9 @@ public void testStrings() throws Exception {
     testJdbcTypeCompatibility("VARCHAR", "foo", new StringProvider());
 
     // Test strings with limited size
-    testJdbcTypeCompatibility("VARCHAR(2)", "", new StringProvider());
+    testJdbcTypeCompatibility("VARCHAR(1)", "", new StringProvider());
     testJdbcTypeCompatibility("VARCHAR(2)", "ab", new StringProvider());
-    expectArrowNotSupported("VARCHAR(2)", "abc");
+    testJdbcTypeCompatibility("VARCHAR(2)", "🍞❄", new StringProvider());
 
     // test booleans
     testJdbcTypeCompatibility("CHAR(5)", true, "true", new BooleanProvider(), new StringProvider());
@@ -62,36 +73,164 @@ public void testStrings() throws Exception {
   }
 
   @Test
-  @Ignore("SNOW-663621")
   public void testNonAsciiStrings() throws Exception {
-    testIngestion(
-        "VARCHAR", "ž, š, č, ř, c, j, ď, ť, ň", "ž, š, č, ř, c, j, ď, ť, ň", new StringProvider());
+    testJdbcTypeCompatibility(
+        "VARCHAR", "❄😃öüß0ö😃üä++ěšíáýšěčí🍞áýřž+šář+🍞ýšš😃čžýříéě+ž❄", new StringProvider());
+  }
+
+  @Test
+  public void testStringCreatedFromInvalidBytes() throws Exception {
+    byte[] bytes = new byte[256];
+    int counter = 0;
+    while (counter < 256) {
+      bytes[counter] = (byte) (Byte.MIN_VALUE + counter);
+      counter++;
+    }
+
+    String s = new String(bytes, StandardCharsets.UTF_8);
+    testJdbcTypeCompatibility("VARCHAR", s, new StringProvider());
   }
 
   @Test
   public void testMaxAllowedString() throws Exception {
-    StringBuilder maxAllowedStringBuilder = buildString('a', 16 * 1024 * 1024);
-    String maxString = maxAllowedStringBuilder.toString();
-    testIngestion("VARCHAR", maxString, maxString, new StringProvider());
-    expectArrowNotSupported("VARCHAR", maxAllowedStringBuilder.append('a').toString());
+    // 1-byte chars
+    String maxString = buildString("a", MB_16);
+    testIngestion("VARCHAR", maxString, new StringProvider());
+    expectArrowNotSupported("VARCHAR", maxString + "a");
+
+    // 2-byte chars
+    maxString = buildString("š", MB_16 / 2);
+    testIngestion("VARCHAR", maxString, new StringProvider());
+
+    expectArrowNotSupported("VARCHAR", maxString + "a");
+
+    // 3-byte chars
+    maxString = buildString("❄", MB_16 / 3);
+    testIngestion("VARCHAR", maxString, new StringProvider());
+    expectArrowNotSupported("VARCHAR", maxString + "aa");
+
+    // 4-byte chars
+    maxString = buildString("🍞", MB_16 / 4);
+    testIngestion("VARCHAR", maxString, new StringProvider());
+    expectArrowNotSupported("VARCHAR", maxString + "a");
   }
 
   @Test
-  @Ignore("SNOW-663621")
-  public void testMaxAllowedMultibyteString() throws Exception {
-    String times16 = "čččččččččččččččč";
-    String times17 = "ččččččččččččččččč";
-    testIngestion("VARCHAR", times16, times16, new StringProvider()); // works fine
-    testIngestion("VARCHAR", times17, times17, new StringProvider()); // fails
-    //    expectArrowNotSupported("VARCHAR",
-    // maxAllowedMultibyteStringBuilder.append('a').toString());
+  public void testPrefixFF() throws Exception {
+
+    // 11x \xFFFF
+    testIngestion(
+        "VARCHAR",
+        "\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF",
+        new StringProvider());
+    // 10x \xFFFF + chars
+    testIngestion(
+        "VARCHAR",
+        "\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFFaaaaaaaaaaaaaaaaaaaaaaaaaa",
+        new StringProvider());
+
+    // chars + 15+ times \uFFFF
+    ingestManyAndMigrate(
+        "aaaaaaaaa\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF");
+
+    // chars + 15+ times \uFFFF + chars
+    ingestManyAndMigrate(
+        "aaaaaaaaa\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFFaaaaaaaaa");
+
+    // 15+ times \uFFFF
+    ingestManyAndMigrate(
+        "\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF");
+
+    // 15+ times \uFFFF + chars
+    ingestManyAndMigrate(
+        "\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFFaaaaaaaaa");
+  }
+
+  @Test
+  public void testMultiByteCharComparison() throws Exception {
+    ingestManyAndMigrate("a", "❄");
+    ingestManyAndMigrate("❄", "a");
+
+    ingestManyAndMigrate(
+        "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+        "❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄");
+    ingestManyAndMigrate(
+        "❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄",
+        "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
   }
 
-  private StringBuilder buildString(char character, int count) {
-    StringBuilder maxStringBuilder = new StringBuilder(count);
-    for (int i = 0; i < count; i++) {
-      maxStringBuilder.append(character);
+  /**
+   * Ingests string with length around EP-truncation point and asserts that both shorter, equal and
+   * longer strings are ingested correctly
+   */
+  @Test
+  public void testTruncationAndIncrementation() throws Exception {
+    // Test 1-byte
+    testIngestion("VARCHAR", buildString("a", 31), new StringProvider());
+    testIngestion("VARCHAR", buildString("a", 32), new StringProvider());
+    testIngestion("VARCHAR", buildString("a", 33), new StringProvider());
+
+    // Test 2-byte
+    testIngestion("VARCHAR", buildString("š", 15), new StringProvider());
+    testIngestion("VARCHAR", buildString("š", 16), new StringProvider());
+    testIngestion("VARCHAR", buildString("š", 17), new StringProvider());
+    testIngestion("VARCHAR", "a" + buildString("š", 15), new StringProvider());
+    testIngestion("VARCHAR", "a" + buildString("š", 16), new StringProvider());
+
+    // Test 3-byte
+    testIngestion("VARCHAR", buildString("❄", 10), new StringProvider());
+    testIngestion("VARCHAR", buildString("❄", 11), new StringProvider());
+    testIngestion("VARCHAR", buildString("❄", 12), new StringProvider());
+
+    // Test 4-byte
+    testIngestion("VARCHAR", buildString("🍞", 6), new StringProvider());
+    testIngestion("VARCHAR", buildString("🍞", 7), new StringProvider());
+    testIngestion("VARCHAR", buildString("🍞", 8), new StringProvider());
+
+    testIngestion("VARCHAR", "a" + buildString("🍞", 7), new StringProvider());
+  }
+
+  @Test
+  @Ignore("Failing due to GS SNOW-690281")
+  public void testByteSplit() throws Exception {
+    testIngestion("VARCHAR", "a" + buildString("🍞", 8), new StringProvider());
+    testIngestion("VARCHAR", "a" + buildString("🍞", 9), new StringProvider());
+  }
+
+  /**
+   * Verifies that non-nullable collated columns are not supported at all and an exception is thrown
+   * already while creating the channel.
+   */
+  @Test
+  public void testCollatedColumnsNotSupported() throws SQLException {
+    String tableName = getRandomIdentifier();
+    conn.createStatement()
+        .execute(
+            String.format(
+                "create or replace table %s (\"create\" string collate 'en-ci')", tableName));
+    try {
+      openChannel(tableName);
+      Assert.fail("Opening a channel shouldn't have succeeded");
+    } catch (SFException e) {
+      Assert.assertEquals(ErrorCode.UNSUPPORTED_DATA_TYPE.getMessageCode(), e.getVendorCode());
     }
-    return maxStringBuilder;
+  }
+
+  /**
+   * Ingest multiple values, wait for the latest offset to be committed, migrate the table and
+   * assert no errors have been thrown.
+   */
+  protected <STREAMING_INGEST_WRITE> void ingestManyAndMigrate(STREAMING_INGEST_WRITE... values)
+      throws Exception {
+    String tableName = createTable("VARCHAR");
+    SnowflakeStreamingIngestChannel channel = openChannel(tableName);
+    String offsetToken = null;
+    for (int i = 0; i < values.length; i++) {
+      offsetToken = String.format("offsetToken%d", i);
+      channel.insertRow(createStreamingIngestRow(values[i]), offsetToken);
+    }
+
+    TestUtils.waitForOffset(channel, offsetToken);
+    migrateTable(tableName); // migration should always succeed
   }
 }

From a97a6f548c0205cc90a17626cb6cd2d265719856 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Thu, 26 Jan 2023 17:16:23 +0100
Subject: [PATCH 116/356] Enable Parquet data type tests (#313)

---
 src/test/java/net/snowflake/ingest/TestUtils.java           | 5 +----
 .../ingest/streaming/internal/StreamingIngestIT.java        | 6 +-----
 .../streaming/internal/datatypes/AbstractDataTypeTest.java  | 6 ------
 3 files changed, 2 insertions(+), 15 deletions(-)

diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 9506e4fac..35a4e0fe4 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -154,10 +154,7 @@ public static Collection<Object[]> getBdecVersionItCases() {
     }
     return Arrays.asList(
         new Object[][] {
-          {"Arrow", Constants.BdecVersion.ONE},
-          // TODO: uncomment once SNOW-659721 is deployed and we set the parameter
-          // DISABLE_PARQUET_CACHE to true for the test account
-          // {"Parquet", Constants.BdecVersion.THREE}
+          {"Arrow", Constants.BdecVersion.ONE}, {"Parquet", Constants.BdecVersion.THREE}
         });
   }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index cf3f6f76a..aa48c07f7 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -77,11 +77,7 @@ public void beforeAll() throws Exception {
     testDb = TEST_DB_PREFIX + "_" + UUID.randomUUID().toString().substring(0, 4);
     // Create a streaming ingest client
     jdbcConnection = TestUtils.getConnection(true);
-    if (bdecVersion == Constants.BdecVersion.THREE) {
-      // TODO: encryption and interleaved mode are not yet supported by server side's Parquet
-      // scanner if local file cache is enabled (SNOW-656500)
-      jdbcConnection.createStatement().execute("alter session set disable_parquet_cache=true;");
-    }
+
     jdbcConnection
         .createStatement()
         .execute(String.format("create or replace database %s;", testDb));
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index d18e735ba..729f3d354 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -77,12 +77,6 @@ public void before() throws Exception {
 
     conn.createStatement().execute(String.format("use warehouse %s;", TestUtils.getWarehouse()));
 
-    if (bdecVersion == Constants.BdecVersion.THREE) {
-      // TODO: encryption and interleaved mode are not yet supported by server side's Parquet
-      // scanner if local file cache is enabled (SNOW-656500)
-      conn.createStatement().execute("alter session set disable_parquet_cache=true;");
-    }
-
     Properties props = TestUtils.getProperties(bdecVersion);
     if (props.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       props.setProperty(ROLE, "ACCOUNTADMIN");

From 0ee8ff30b09cddcca534bcb0608bf093758d51d7 Mon Sep 17 00:00:00 2001
From: sfc-gh-wfateem <waleed.fateem@snowflake.com>
Date: Wed, 1 Feb 2023 16:40:28 -0600
Subject: [PATCH 117/356] [SNOW-726993] Ingest SDK Does Not Honor
 http.nonProxyHosts JVM Argument

The JDBC driver was upgraded to 3.13.18 and the HttpUtil code was changed to
take into account that a user may need to bypass the proxy by providing
the -Dhttp.nonProxyHosts JVM argument
---
 pom.xml                                       |  2 +-
 .../snowflake/ingest/SimpleIngestManager.java |  2 +-
 ...nowflakeStreamingIngestClientInternal.java |  5 +-
 .../net/snowflake/ingest/utils/HttpUtil.java  | 47 +++++++++++++++++--
 .../internal/StreamingIngestStageTest.java    | 45 ++++++++++++++++++
 5 files changed, 93 insertions(+), 8 deletions(-)

diff --git a/pom.xml b/pom.xml
index a844c7ea4..4fec9dbc4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -194,7 +194,7 @@
         <dependency>
             <groupId>net.snowflake</groupId>
             <artifactId>snowflake-jdbc</artifactId>
-            <version>3.13.15</version>
+            <version>3.13.18</version>
         </dependency>
 
         <!-- jwt token for key pair authentication with GS -->
diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
index e5163a4dc..d167488dd 100644
--- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
+++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
@@ -407,7 +407,7 @@ private void init(String account, String user, String pipe, KeyPair keyPair) {
     this.keyPair = keyPair;
 
     // make our client for sending requests
-    httpClient = HttpUtil.getHttpClient();
+    httpClient = HttpUtil.getHttpClient(account);
     // make the request builder we'll use to build messages to the service
   }
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 3e6ae6614..5800724c5 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -101,6 +101,8 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
   // Name of the client
   private final String name;
 
+  private String accountName;
+
   // Snowflake role for the client to use
   private String role;
 
@@ -165,8 +167,9 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
     this.parameterProvider = new ParameterProvider(parameterOverrides, prop);
 
     this.name = name;
+    this.accountName = accountURL.getAccount();
     this.isTestMode = isTestMode;
-    this.httpClient = httpClient == null ? HttpUtil.getHttpClient() : httpClient;
+    this.httpClient = httpClient == null ? HttpUtil.getHttpClient(accountName) : httpClient;
     this.channelCache = new ChannelCache<>();
     this.allocator = new RootAllocator();
     this.isClosed = false;
diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index 3b34459f5..db513708c 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -14,6 +14,7 @@
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
 import javax.net.ssl.SSLContext;
 import net.snowflake.client.core.SFSessionProperty;
 import net.snowflake.client.jdbc.internal.apache.http.HttpHost;
@@ -47,9 +48,10 @@ public class HttpUtil {
   public static final String USE_PROXY = "http.useProxy";
   public static final String PROXY_HOST = "http.proxyHost";
   public static final String PROXY_PORT = "http.proxyPort";
-
+  public static final String NON_PROXY_HOSTS = "http.nonProxyHosts";
   public static final String HTTP_PROXY_USER = "http.proxyUser";
   public static final String HTTP_PROXY_PASSWORD = "http.proxyPassword";
+  private static final String SNOWFLAKE_DOMAIN_NAME = ".snowflakecomputing.com";
 
   private static final String PROXY_SCHEME = "http";
   private static final String FIRST_FAULT_TIMESTAMP = "FIRST_FAULT_TIMESTAMP";
@@ -84,11 +86,15 @@ public class HttpUtil {
   // Only connections that are currently owned, not checked out, are subject to idle timeouts.
   private static final int DEFAULT_IDLE_CONNECTION_TIMEOUT_SECONDS = 30;
 
-  public static CloseableHttpClient getHttpClient() {
+  /**
+   * @param {@code String} account name to connect to (excluding snowflakecomputing.com domain)
+   * @return Instance of CloseableHttpClient
+   */
+  public static CloseableHttpClient getHttpClient(String accountName) {
     if (httpClient == null) {
       synchronized (HttpUtil.class) {
         if (httpClient == null) {
-          initHttpClient();
+          initHttpClient(accountName);
         }
       }
     }
@@ -100,7 +106,7 @@ public static CloseableHttpClient getHttpClient() {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(HttpUtil.class);
 
-  private static void initHttpClient() {
+  private static void initHttpClient(String accountName) {
 
     Security.setProperty("ocsp.enable", "true");
 
@@ -146,7 +152,7 @@ private static void initHttpClient() {
             .setDefaultRequestConfig(requestConfig);
 
     // proxy settings
-    if ("true".equalsIgnoreCase(System.getProperty(USE_PROXY))) {
+    if ("true".equalsIgnoreCase(System.getProperty(USE_PROXY)) && !shouldBypassProxy(accountName)) {
       if (System.getProperty(PROXY_PORT) == null) {
         throw new IllegalArgumentException(
             "proxy port number is not provided, please assign proxy port to http.proxyPort option");
@@ -316,6 +322,12 @@ public static Properties generateProxyPropertiesForJDBC() {
         proxyProperties.put(SFSessionProperty.PROXY_USER.getPropertyKey(), proxyUser);
         proxyProperties.put(SFSessionProperty.PROXY_PASSWORD.getPropertyKey(), proxyPassword);
       }
+
+      // Check if http.nonProxyHosts was set
+      final String nonProxyHosts = System.getProperty(NON_PROXY_HOSTS);
+      if (!isNullOrEmpty(nonProxyHosts)) {
+        proxyProperties.put(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey(), nonProxyHosts);
+      }
     }
     return proxyProperties;
   }
@@ -401,4 +413,29 @@ private static String createPoolStatsInfo(String title, PoolStats poolStats) {
     }
     return title;
   }
+
+  // Changes the account name to the format accountName.snowflakecomputing.com
+  // then returns a boolean to indicate if we should go through a proxy or not.
+  public static Boolean shouldBypassProxy(String accountName) {
+    String targetHost = accountName + SNOWFLAKE_DOMAIN_NAME;
+    return System.getProperty(NON_PROXY_HOSTS) != null && isInNonProxyHosts(targetHost);
+  }
+
+  /* The target hostname input is compared with the hosts in the '|' separated
+    list provided by the http.nonProxyHosts parameter using regex. The nonProxyHosts
+    will be used as our Patterns, so we need to replace the '.' and '*' characters
+    since those are special regex constructs that mean 'any character,' and
+    'repeat 0 or more times.'
+  */
+  private static Boolean isInNonProxyHosts(String targetHost) {
+    String nonProxyHosts =
+        System.getProperty(NON_PROXY_HOSTS).replace(".", "\\.").replace("*", ".*");
+    String[] nonProxyHostsArray = nonProxyHosts.split("\\|");
+    for (String i : nonProxyHostsArray) {
+      if (Pattern.compile(i).matcher(targetHost).matches()) {
+        return true;
+      }
+    }
+    return false;
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
index 79c809e24..45cd82c33 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
@@ -3,10 +3,12 @@
 import static net.snowflake.client.core.Constants.CLOUD_STORAGE_CREDENTIALS_EXPIRED;
 import static net.snowflake.ingest.utils.HttpUtil.HTTP_PROXY_PASSWORD;
 import static net.snowflake.ingest.utils.HttpUtil.HTTP_PROXY_USER;
+import static net.snowflake.ingest.utils.HttpUtil.NON_PROXY_HOSTS;
 import static net.snowflake.ingest.utils.HttpUtil.PROXY_HOST;
 import static net.snowflake.ingest.utils.HttpUtil.PROXY_PORT;
 import static net.snowflake.ingest.utils.HttpUtil.USE_PROXY;
 import static net.snowflake.ingest.utils.HttpUtil.generateProxyPropertiesForJDBC;
+import static net.snowflake.ingest.utils.HttpUtil.shouldBypassProxy;
 import static org.mockito.Mockito.times;
 
 import java.io.ByteArrayInputStream;
@@ -391,11 +393,13 @@ public void testGenerateProxyPropertiesForJDBC() {
     String oldProxyPort = System.getProperty(PROXY_PORT);
     String oldUser = System.getProperty(HTTP_PROXY_USER);
     String oldPassword = System.getProperty(HTTP_PROXY_PASSWORD);
+    String oldNonProxyHosts = System.getProperty(NON_PROXY_HOSTS);
 
     String proxyHost = "localhost";
     String proxyPort = "8080";
     String user = "admin";
     String password = "test";
+    String nonProxyHosts = "*.snowflakecomputing.com";
 
     try {
       // Test empty properties when USE_PROXY is NOT set;
@@ -407,6 +411,7 @@ public void testGenerateProxyPropertiesForJDBC() {
       System.setProperty(PROXY_PORT, proxyPort);
       System.setProperty(HTTP_PROXY_USER, user);
       System.setProperty(HTTP_PROXY_PASSWORD, password);
+      System.setProperty(NON_PROXY_HOSTS, nonProxyHosts);
 
       // Verify that properties are set
       props = generateProxyPropertiesForJDBC();
@@ -415,6 +420,8 @@ public void testGenerateProxyPropertiesForJDBC() {
       Assert.assertEquals(proxyPort, props.get(SFSessionProperty.PROXY_PORT.getPropertyKey()));
       Assert.assertEquals(user, props.get(SFSessionProperty.PROXY_USER.getPropertyKey()));
       Assert.assertEquals(password, props.get(SFSessionProperty.PROXY_PASSWORD.getPropertyKey()));
+      Assert.assertEquals(
+          nonProxyHosts, props.get(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey()));
     } finally {
       // Cleanup
       if (oldUseProxy != null) {
@@ -426,6 +433,44 @@ public void testGenerateProxyPropertiesForJDBC() {
         System.setProperty(HTTP_PROXY_USER, oldUser);
         System.setProperty(HTTP_PROXY_PASSWORD, oldPassword);
       }
+      if (oldNonProxyHosts != null) {
+        System.setProperty(NON_PROXY_HOSTS, oldNonProxyHosts);
+      }
+    }
+  }
+
+  @Test
+  public void testShouldBypassProxy() {
+    String oldNonProxyHosts = System.getProperty(NON_PROXY_HOSTS);
+    String accountName = "accountName12345";
+    String accountDashName = "account-name12345";
+    String accountUnderscoreName = "account_name12345";
+    String nonProxyHosts = "*.snowflakecomputing.com|localhost";
+
+    System.setProperty(NON_PROXY_HOSTS, nonProxyHosts);
+    Assert.assertTrue(shouldBypassProxy(accountName));
+    Assert.assertTrue(shouldBypassProxy(accountDashName));
+    Assert.assertTrue(shouldBypassProxy(accountUnderscoreName));
+
+    String accountNamePrivateLink = "accountName12345.privatelink";
+    nonProxyHosts = "*.privatelink.snowflakecomputing.com|localhost";
+    System.setProperty(NON_PROXY_HOSTS, nonProxyHosts);
+    Assert.assertTrue(shouldBypassProxy(accountNamePrivateLink));
+
+    // Previous tests should return false with the new nonProxyHosts value
+    Assert.assertFalse(shouldBypassProxy(accountName));
+    Assert.assertFalse(shouldBypassProxy(accountDashName));
+    Assert.assertFalse(shouldBypassProxy(accountUnderscoreName));
+
+    // All tests should return false after clearing the nonProxyHosts property
+    System.clearProperty(NON_PROXY_HOSTS);
+    Assert.assertFalse(shouldBypassProxy(accountName));
+    Assert.assertFalse(shouldBypassProxy(accountDashName));
+    Assert.assertFalse(shouldBypassProxy(accountUnderscoreName));
+    Assert.assertFalse(shouldBypassProxy(accountNamePrivateLink));
+
+    if (oldNonProxyHosts != null) {
+      System.setProperty(NON_PROXY_HOSTS, oldNonProxyHosts);
     }
   }
 }

From b2a38d40d264499d20b6f92f49c0522dcdf76457 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 1 Feb 2023 23:17:02 +0000
Subject: [PATCH 118/356] fix test failures

---
 ...SnowflakeStreamingIngestClientInternal.java |  2 +-
 .../net/snowflake/ingest/utils/HttpUtil.java   | 18 ++++++++++--------
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 5800724c5..05ef85322 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -167,7 +167,7 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
     this.parameterProvider = new ParameterProvider(parameterOverrides, prop);
 
     this.name = name;
-    this.accountName = accountURL.getAccount();
+    this.accountName = accountURL == null ? null : accountURL.getAccount();
     this.isTestMode = isTestMode;
     this.httpClient = httpClient == null ? HttpUtil.getHttpClient(accountName) : httpClient;
     this.channelCache = new ChannelCache<>();
diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index db513708c..72bf32767 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -414,19 +414,21 @@ private static String createPoolStatsInfo(String title, PoolStats poolStats) {
     return title;
   }
 
-  // Changes the account name to the format accountName.snowflakecomputing.com
-  // then returns a boolean to indicate if we should go through a proxy or not.
+  /**
+   * Changes the account name to the format accountName.snowflakecomputing.com then returns a
+   * boolean to indicate if we should go through a proxy or not.
+   */
   public static Boolean shouldBypassProxy(String accountName) {
     String targetHost = accountName + SNOWFLAKE_DOMAIN_NAME;
     return System.getProperty(NON_PROXY_HOSTS) != null && isInNonProxyHosts(targetHost);
   }
 
-  /* The target hostname input is compared with the hosts in the '|' separated
-    list provided by the http.nonProxyHosts parameter using regex. The nonProxyHosts
-    will be used as our Patterns, so we need to replace the '.' and '*' characters
-    since those are special regex constructs that mean 'any character,' and
-    'repeat 0 or more times.'
-  */
+  /**
+   * The target hostname input is compared with the hosts in the '|' separated list provided by the
+   * http.nonProxyHosts parameter using regex. The nonProxyHosts will be used as our Patterns, so we
+   * need to replace the '.' and '*' characters since those are special regex constructs that mean
+   * 'any character,' and 'repeat 0 or more times.'
+   */
   private static Boolean isInNonProxyHosts(String targetHost) {
     String nonProxyHosts =
         System.getProperty(NON_PROXY_HOSTS).replace(".", "\\.").replace("*", ".*");

From 227bb2ca0b7e5fe99646886d91678e86bc0b2744 Mon Sep 17 00:00:00 2001
From: Louis Zhang <louis.zhang@snowflake.com>
Date: Thu, 2 Feb 2023 13:15:12 -0800
Subject: [PATCH 119/356] SNOW-731571 SimpleIngestIT runtime created database

Description
Each time running SimpleIngestIT, we create a new database instead of using the pre-existing database.

Testing
Integration Test.
---
 .../java/net/snowflake/ingest/SimpleIngestIT.java  | 13 +++++++++++++
 src/test/java/net/snowflake/ingest/TestUtils.java  | 14 ++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
index 312d127b9..e77777269 100644
--- a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
@@ -82,6 +82,19 @@ public void beforeAll() throws Exception {
 
     stageWithPatternName = "ingest_sdk_test_stage_pattern" + RAND_NUM;
 
+
+    String databaseName = TestUtils.getDatabase();
+
+    String schemaName = TestUtils.getSchema();
+
+    TestUtils.executeQuery("create database if not exists " + databaseName);
+
+    TestUtils.executeQuery("create schema if not exists " + schemaName);
+
+    TestUtils.executeQuery("use database " + databaseName);
+
+    TestUtils.executeQuery("use schema " + schemaName);
+
     TestUtils.executeQuery("create or replace table " + tableName + " (str string, num int)");
 
     TestUtils.executeQuery("create or replace stage " + stageName);
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 35a4e0fe4..12f114a54 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -209,6 +209,20 @@ public static KeyPair getKeyPair() throws Exception {
     return keyPair;
   }
 
+  public static String getDatabase() throws Exception {
+    if (profile == null) {
+      init();
+    }
+    return database;
+  }
+
+  public static String getSchema() throws Exception {
+    if (profile == null) {
+      init();
+    }
+    return schema;
+  }
+
   public static Properties getProperties(Constants.BdecVersion bdecVersion) throws Exception {
     if (profile == null) {
       init();

From 84739242e6ebcb47887791a4c89180a2b16c7f7a Mon Sep 17 00:00:00 2001
From: Louis Zhang <louis.zhang@snowflake.com>
Date: Thu, 2 Feb 2023 13:51:59 -0800
Subject: [PATCH 120/356] reformat the code

---
 src/test/java/net/snowflake/ingest/SimpleIngestIT.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
index e77777269..0617898c8 100644
--- a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
@@ -82,7 +82,6 @@ public void beforeAll() throws Exception {
 
     stageWithPatternName = "ingest_sdk_test_stage_pattern" + RAND_NUM;
 
-
     String databaseName = TestUtils.getDatabase();
 
     String schemaName = TestUtils.getSchema();

From a7a78def87d982705b9d4953285f451f56877652 Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Mon, 6 Feb 2023 22:35:51 +0100
Subject: [PATCH 121/356] SNOW-620624 enforce single rowgroup Parquet files
 (#324)

SNOW-620624 force Parquet generator to produce dingle rowgroup, single-page files even when blob size is too big
---
 .../parquet/hadoop/BdecParquetWriter.java     |  35 ++-
 .../java/net/snowflake/ingest/TestUtils.java  | 100 +++++++++
 .../internal/StreamingIngestBigFilesIT.java   | 199 ++++++++++++++++++
 3 files changed, 326 insertions(+), 8 deletions(-)
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java

diff --git a/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java b/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java
index a353af7e3..9122e20be 100644
--- a/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java
+++ b/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java
@@ -66,7 +66,7 @@ public BdecParquetWriter(
             file,
             schema,
             ParquetFileWriter.Mode.CREATE,
-            ParquetWriter.DEFAULT_BLOCK_SIZE,
+            Constants.MAX_BLOB_SIZE_IN_BYTES * 2,
             ParquetWriter.MAX_PADDING_SIZE_DEFAULT,
             encodingProps.getColumnIndexTruncateLength(),
             encodingProps.getStatisticsTruncateLength(),
@@ -94,7 +94,7 @@ public BdecParquetWriter(
             writeSupport,
             schema,
             writeContext.getExtraMetaData(),
-            ParquetWriter.DEFAULT_BLOCK_SIZE,
+            Constants.MAX_BLOB_SIZE_IN_BYTES * 2,
             compressor,
             true,
             encodingProps);
@@ -120,6 +120,30 @@ public void close() throws IOException {
   }
 
   private static ParquetProperties createParquetProperties() {
+    /**
+     * There are two main limitations on the server side that we have to overcome by tweaking
+     * Parquet limits:
+     *
+     * <p>1. Scanner supports only the case when the row number in pages is the same. Remember that
+     * a page has data from only one column.
+     *
+     * <p>2. Scanner supports only 1 row group per Parquet file.
+     *
+     * <p>We can't guarantee that each page will have the same number of rows, because we have no
+     * internal control over Parquet lib. That's why to satisfy 1., we will generate one page per
+     * column.
+     *
+     * <p>To satisfy 1. and 2., we will disable a check that decides when to flush buffered rows to
+     * row groups and pages. The check happens after a configurable amount of row counts and by
+     * setting it to Integer.MAX_VALUE, Parquet lib will never perform the check and flush all
+     * buffered rows to one row group on close(). The same check decides when to flush a rowgroup to
+     * page. So by disabling it, we are not checking any page limits as well and flushing all
+     * buffered data of the rowgroup to one page.
+     *
+     * <p>TODO: Remove the enforcements of single row group SNOW-738040 and single page (per column)
+     * SNOW-737331. TODO: Revisit block and page size estimate after limitation (1) is removed
+     * SNOW-738614 *
+     */
     return ParquetProperties.builder()
         // PARQUET_2_0 uses Encoding.DELTA_BYTE_ARRAY for byte arrays (e.g. SF sb16)
         // server side does not support it TODO: SNOW-657238
@@ -129,13 +153,8 @@ private static ParquetProperties createParquetProperties() {
         // the dictionary encoding (Encoding.*_DICTIONARY) is not supported by server side
         // scanner yet
         .withDictionaryEncoding(false)
-
-        // Historically server side scanner supports only the case when the row number in all
-        // pages is the same.
-        // The quick fix is to effectively disable the page size/row limit
-        // to always have one page per chunk until server side is generalised.
-        .withPageSize((int) Constants.MAX_CHUNK_SIZE_IN_BYTES * 2)
         .withPageRowCountLimit(Integer.MAX_VALUE)
+        .withMinRowCountForPageSizeCheck(Integer.MAX_VALUE)
         .build();
   }
 
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 12f114a54..2e1f0cb1d 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -16,6 +16,7 @@
 import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION;
 
 import java.io.IOException;
+import java.math.BigDecimal;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -27,14 +28,20 @@
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Properties;
+import java.util.Random;
+import java.util.function.Supplier;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
+import net.snowflake.ingest.streaming.InsertValidationResponse;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.Utils;
@@ -384,6 +391,48 @@ public static void waitForOffset(SnowflakeStreamingIngestChannel channel, String
             expectedOffset, lastCommittedOffset));
   }
 
+  public static void waitChannelFlushed(SnowflakeStreamingIngestChannel channel, int numberOfRows) {
+    String latestCommittedOffsetToken = null;
+    for (int i = 1; i < 40; i++) {
+      latestCommittedOffsetToken = channel.getLatestCommittedOffsetToken();
+      if (latestCommittedOffsetToken != null
+          && latestCommittedOffsetToken.equals(Integer.toString(numberOfRows - 1))) {
+        return;
+      }
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException e) {
+        throw new RuntimeException(
+            "Interrupted waitChannelFlushed for " + numberOfRows + " rows", e);
+      }
+    }
+    Assert.fail(
+        "Row sequencer not updated before timeout, latestCommittedOffsetToken: "
+            + latestCommittedOffsetToken);
+  }
+
+  /** Verify the insert validation response and throw the exception if needed */
+  public static void verifyInsertValidationResponse(InsertValidationResponse response) {
+    if (response.hasErrors()) {
+      throw response.getInsertErrors().get(0).getException();
+    }
+  }
+
+  public static void verifyTableRowCount(
+      int rowNumber, Connection jdbcConnection, String database, String schema, String tableName) {
+    try {
+      ResultSet resultCount =
+          jdbcConnection
+              .createStatement()
+              .executeQuery(
+                  String.format("select count(*) from %s.%s.%s", database, schema, tableName));
+      resultCount.next();
+      Assert.assertEquals(rowNumber, resultCount.getLong(1));
+    } catch (SQLException e) {
+      throw new RuntimeException("Cannot verifyTableRowCount for " + tableName, e);
+    }
+  }
+
   /**
    * Creates a string from a certain number of concatenated strings e.g. buildString("ab", 2) =>
    * abab
@@ -395,4 +444,55 @@ public static String buildString(String str, int count) {
     }
     return sb.toString();
   }
+
+  public static Map<String, Object> getRandomRow(Random r, boolean nullable) {
+    Map<String, Object> row = new HashMap<>();
+
+    row.put("num_2_1", nullOrIfNullable(nullable, r, () -> r.nextInt(100) / 10.0));
+    row.put("num_4_2", nullOrIfNullable(nullable, r, () -> r.nextInt(10000) / 100.0));
+    row.put(
+        "num_9_4", nullOrIfNullable(nullable, r, () -> r.nextInt(1000000000) / Math.pow(10, 4)));
+    row.put(
+        "num_18_7",
+        nullOrIfNullable(nullable, r, () -> nextLongOfPrecision(r, 18) / Math.pow(10, 7)));
+    row.put(
+        "num_38_15",
+        nullOrIfNullable(
+            nullable,
+            r,
+            () ->
+                new BigDecimal(
+                    "" + nextLongOfPrecision(r, 18) + "." + Math.abs(nextLongOfPrecision(r, 15)))));
+
+    row.put("num_float", nullOrIfNullable(nullable, r, () -> nextFloat(r)));
+    row.put("str", nullOrIfNullable(nullable, r, () -> nextString(r)));
+    row.put("bin", nullOrIfNullable(nullable, r, () -> nextBytes(r)));
+
+    return row;
+  }
+
+  private static <T> T nullOrIfNullable(boolean nullable, Random r, Supplier<T> value) {
+    return !nullable ? value.get() : (r.nextBoolean() ? value.get() : null);
+  }
+
+  private static long nextLongOfPrecision(Random r, int precision) {
+    return r.nextLong() % Math.round(Math.pow(10, precision));
+  }
+
+  private static String nextString(Random r) {
+    return new String(nextBytes(r));
+  }
+
+  private static byte[] nextBytes(Random r) {
+    byte[] bin = new byte[128];
+    r.nextBytes(bin);
+    for (int i = 0; i < bin.length; i++) {
+      bin[i] = (byte) (Math.abs(bin[i]) % 25 + 97); // ascii letters
+    }
+    return bin;
+  }
+
+  private static double nextFloat(Random r) {
+    return (r.nextLong() % Math.round(Math.pow(10, 10))) / 100000d;
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
new file mode 100644
index 000000000..f12484062
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
@@ -0,0 +1,199 @@
+package net.snowflake.ingest.streaming.internal;
+
+import static net.snowflake.ingest.TestUtils.verifyTableRowCount;
+import static net.snowflake.ingest.utils.Constants.ROLE;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.concurrent.*;
+import net.snowflake.ingest.TestUtils;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import net.snowflake.ingest.utils.Constants;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+/** Ingest large amount of rows. */
+@RunWith(Parameterized.class)
+public class StreamingIngestBigFilesIT {
+  @Parameterized.Parameters(name = "{0}")
+  public static Collection<Object[]> bdecVersion() {
+    return TestUtils.getBdecVersionItCases();
+  }
+
+  private static final String TEST_DB_PREFIX = "STREAMING_INGEST_TEST_DB";
+  private static final String TEST_SCHEMA = "STREAMING_INGEST_TEST_SCHEMA";
+
+  private Properties prop;
+
+  private SnowflakeStreamingIngestClientInternal<?> client;
+  private Connection jdbcConnection;
+  private String testDb;
+
+  private final Constants.BdecVersion bdecVersion;
+
+  public StreamingIngestBigFilesIT(
+      @SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
+    this.bdecVersion = bdecVersion;
+  }
+
+  @Before
+  public void beforeAll() throws Exception {
+    testDb = TEST_DB_PREFIX + "_" + UUID.randomUUID().toString().substring(0, 4);
+    // Create a streaming ingest client
+    jdbcConnection = TestUtils.getConnection(true);
+
+    jdbcConnection
+        .createStatement()
+        .execute(String.format("create or replace database %s;", testDb));
+    jdbcConnection
+        .createStatement()
+        .execute(String.format("create or replace schema %s.%s;", testDb, TEST_SCHEMA));
+    // Set timezone to UTC
+    jdbcConnection.createStatement().execute("alter session set timezone = 'UTC';");
+    jdbcConnection
+        .createStatement()
+        .execute(String.format("use warehouse %s", TestUtils.getWarehouse()));
+
+    prop = TestUtils.getProperties(bdecVersion);
+    if (prop.getProperty(ROLE).equals("DEFAULT_ROLE")) {
+      prop.setProperty(ROLE, "ACCOUNTADMIN");
+    }
+    client =
+        (SnowflakeStreamingIngestClientInternal<?>)
+            SnowflakeStreamingIngestClientFactory.builder("client1").setProperties(prop).build();
+  }
+
+  @After
+  public void afterAll() throws Exception {
+    client.close();
+    jdbcConnection.createStatement().execute(String.format("drop database %s", testDb));
+  }
+
+  @Test
+  public void testManyRowsMultipleChannelsToMultipleTable()
+      throws SQLException, ExecutionException, InterruptedException {
+    String tableNamePrefix = "t_big_table_";
+
+    int numTables = 2;
+    int numChannels = 4; // channels are assigned round-robin to tables.
+    int batchSize = 20000;
+    int numBatches = 10; // number of rows PER CHANNEL is batchSize * numBatches
+    boolean isNullable = false;
+
+    Map<Integer, Integer> tableIdToNumChannels = new HashMap<>();
+    for (int i = 0; i < numChannels; i++) {
+      tableIdToNumChannels.put(
+          i % numTables, tableIdToNumChannels.getOrDefault(i % numTables, 0) + 1);
+    }
+    for (int i = 0; i < numTables; i++) {
+      String tableName = tableNamePrefix + i;
+      createTableForTest(tableName);
+    }
+
+    ingestRandomRowsToTable(
+        tableNamePrefix, numTables, numChannels, batchSize, numBatches, isNullable);
+
+    for (int i = 0; i < numTables; i++) {
+      int numChannelsToTable = tableIdToNumChannels.get(i);
+      verifyTableRowCount(
+          batchSize * numBatches * numChannelsToTable,
+          jdbcConnection,
+          testDb,
+          TEST_SCHEMA,
+          tableNamePrefix + i);
+
+      // select * to ensure scanning works
+      ResultSet result =
+          jdbcConnection
+              .createStatement()
+              .executeQuery(
+                  String.format(
+                      "select * from %s.%s.%s", testDb, TEST_SCHEMA, tableNamePrefix + i));
+      result.next();
+      Assert.assertNotNull(result.getString("STR"));
+    }
+  }
+
+  private void ingestRandomRowsToTable(
+      String tablePrefix,
+      int numTables,
+      int numChannels,
+      int batchSize,
+      int iterations,
+      boolean isNullable)
+      throws ExecutionException, InterruptedException {
+
+    List<Map<String, Object>> rows = new ArrayList<>();
+    for (int i = 0; i < batchSize; i++) {
+      Random r = new Random();
+      rows.add(TestUtils.getRandomRow(r, isNullable));
+    }
+
+    ExecutorService testThreadPool = Executors.newFixedThreadPool(numChannels);
+    CompletableFuture[] futures = new CompletableFuture[numChannels];
+    List<SnowflakeStreamingIngestChannel> channelList = new ArrayList<>();
+    for (int i = 0; i < numChannels; i++) {
+      final String channelName = "CHANNEL" + i;
+      int finalI = i;
+      futures[i] =
+          CompletableFuture.runAsync(
+              () -> {
+                int targetTable = finalI % numTables;
+                SnowflakeStreamingIngestChannel channel =
+                    openChannel(tablePrefix + targetTable, channelName);
+                channelList.add(channel);
+                for (int val = 0; val < iterations; val++) {
+                  TestUtils.verifyInsertValidationResponse(
+                      channel.insertRows(rows, Integer.toString(val)));
+                }
+              },
+              testThreadPool);
+    }
+    CompletableFuture joined = CompletableFuture.allOf(futures);
+    joined.get();
+    channelList.forEach(channel -> TestUtils.waitChannelFlushed(channel, iterations));
+    testThreadPool.shutdown();
+  }
+
+  private void createTableForTest(String tableName) {
+    try {
+      jdbcConnection
+          .createStatement()
+          .execute(
+              String.format(
+                  "create or replace table %s (\n"
+                      + "                                    num_2_1 NUMBER(2, 1),\n"
+                      + "                                    num_4_2 NUMBER(4, 2),\n"
+                      + "                                    num_9_4 NUMBER(9, 4),\n"
+                      + "                                    num_18_7 NUMBER(18, 7),\n"
+                      + "                                    num_38_15 NUMBER(38, 15),\n"
+                      + "                                    num_float FLOAT,\n"
+                      + "                                    str VARCHAR(256),\n"
+                      + "                                    bin BINARY(256));",
+                  tableName));
+    } catch (SQLException e) {
+      throw new RuntimeException("Cannot create table " + tableName, e);
+    }
+  }
+
+  private SnowflakeStreamingIngestChannel openChannel(String tableName, String channelName) {
+    OpenChannelRequest request =
+        OpenChannelRequest.builder(channelName)
+            .setDBName(testDb)
+            .setSchemaName(TEST_SCHEMA)
+            .setTableName(tableName)
+            .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+            .build();
+
+    // Open a streaming ingest channel from the given client
+    return client.openChannel(request);
+  }
+}

From 5a728ca7c0bc0e39f90964975ce61d1191128836 Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Tue, 7 Feb 2023 10:56:58 +0100
Subject: [PATCH 122/356] SNOW-733699 make Parquet the default blob format
 version instead of Arrow (#322)

---
 .../ingest/utils/ParameterProvider.java       |  3 ++-
 .../java/net/snowflake/ingest/TestUtils.java  |  9 --------
 .../SnowflakeStreamingIngestChannelTest.java  | 21 ++++++++++++++-----
 3 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
index acfa8e7bb..07bca85cc 100644
--- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
+++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -30,7 +30,8 @@ public class ParameterProvider {
   public static final long INSERT_THROTTLE_INTERVAL_IN_MILLIS_DEFAULT = 1000;
   public static final int INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_DEFAULT = 10;
   public static final boolean SNOWPIPE_STREAMING_METRICS_DEFAULT = false;
-  public static final Constants.BdecVersion BLOB_FORMAT_VERSION_DEFAULT = Constants.BdecVersion.ONE;
+  public static final Constants.BdecVersion BLOB_FORMAT_VERSION_DEFAULT =
+      Constants.BdecVersion.THREE;
   public static final int IO_TIME_CPU_RATIO_DEFAULT = 2;
   public static final int BLOB_UPLOAD_MAX_RETRY_COUNT_DEFAULT = 24;
   public static final long MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT = -1L;
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 2e1f0cb1d..293ece5bf 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -150,15 +150,6 @@ private static String getTestProfilePath() {
 
   /** @return list of Bdec versions for which to execute IT tests. */
   public static Collection<Object[]> getBdecVersionItCases() {
-    boolean enableParquetTests =
-        System.getProperty("enableParquetTests") != null
-            && Boolean.parseBoolean(System.getProperty("enableParquetTests"));
-    if (enableParquetTests) {
-      return Arrays.asList(
-          new Object[][] {
-            {"Arrow", Constants.BdecVersion.ONE}, {"Parquet", Constants.BdecVersion.THREE}
-          });
-    }
     return Arrays.asList(
         new Object[][] {
           {"Arrow", Constants.BdecVersion.ONE}, {"Parquet", Constants.BdecVersion.THREE}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 8b731112a..bb328aa08 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -463,9 +463,16 @@ public void testOpenChannelSuccessResponse() throws Exception {
 
   @Test
   public void testInsertRow() {
-    SnowflakeStreamingIngestClientInternal<VectorSchemaRoot> client =
-        new SnowflakeStreamingIngestClientInternal<>("client");
-    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> channel =
+    SnowflakeStreamingIngestClientInternal<?> client;
+    boolean isArrowDefault =
+        ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT == Constants.BdecVersion.ONE;
+    if (isArrowDefault) {
+      client = new SnowflakeStreamingIngestClientInternal<VectorSchemaRoot>("client_ARROW");
+    } else {
+      client = new SnowflakeStreamingIngestClientInternal<ParquetChunkData>("client_PARQUET");
+    }
+
+    SnowflakeStreamingIngestChannelInternal<?> channel =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel",
             "db",
@@ -494,7 +501,7 @@ public void testInsertRow() {
     row.put("col", 1);
 
     // Get data before insert to verify that there is no row (data should be null)
-    ChannelData<VectorSchemaRoot> data = channel.getData("my_snowpipe_streaming.bdec");
+    ChannelData<?> data = channel.getData("my_snowpipe_streaming.bdec");
     Assert.assertNull(data);
 
     long insertStartTimeInMs = System.currentTimeMillis();
@@ -508,7 +515,11 @@ public void testInsertRow() {
     data = channel.getData("my_snowpipe_streaming.bdec");
     Assert.assertEquals(2, data.getRowCount());
     Assert.assertEquals((Long) 1L, data.getRowSequencer());
-    Assert.assertEquals(1, data.getVectors().getFieldVectors().size());
+    Assert.assertEquals(
+        1,
+        isArrowDefault
+            ? ((ChannelData<VectorSchemaRoot>) data).getVectors().getFieldVectors().size()
+            : ((ChannelData<ParquetChunkData>) data).getVectors().rows.get(0).size());
     Assert.assertEquals("2", data.getOffsetToken());
     Assert.assertTrue(data.getBufferSize() > 0);
     Assert.assertTrue(insertStartTimeInMs <= data.getMinMaxInsertTimeInMs().getFirst());

From 8021d106305b2beb0b76c5556c112a74371f57e6 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Wed, 8 Feb 2023 18:21:04 +0100
Subject: [PATCH 123/356] SNOW-711299 New date/time processing (#320)

SnowflakeDateTimeFormat has some issues.
This PR implements the following changes to our date/time processing:

* We no longer depend on `SnowflakeDateTimeFormat`, custom validation and serialization of date/time values has been implemented.
    * `java.time.OffsetDateTime` is the canonical representation of timestamps in the SDK.
    * The following date/time values are going to be accepted:
        * Objects from `java.time` package (`LocalDate`, `LocalDateTime`, `OffsetDateTime`, `ZonedDateTime`, `Instant`, `LocalTime`, `OffsetTime`)
        * Strings in ISO format
        * Integer-stored strings
* Input values for TIMESTAMP_TZ and TIMESTAMP_LTZ, which do not carry timezone information, are by default interpreted in UTC.
    * This can now be changed by calling `OpenChannelRequestBuilder#setDefaultTimezone`.
    * Just like in Snowflake, integer-stored strings (and java.time.Instant in our case) are always interpreted in UTC.
* Detailed Javadoc for insertRow() has been added. It describes accepted values for all Snowflake types, so that users don't have to search the documentation.
* Comprehensive suite of integration tests has been added.

This PR fixes SNOW-663646, as well.
---
 .../ingest/streaming/OpenChannelRequest.java  |   23 +
 .../SnowflakeStreamingIngestChannel.java      |  149 +-
 .../streaming/internal/AbstractRowBuffer.java |    8 +
 .../streaming/internal/ArrowRowBuffer.java    |   75 +-
 .../internal/DataValidationUtil.java          |  379 ++--
 .../streaming/internal/ParquetRowBuffer.java  |   13 +-
 .../internal/ParquetValueParser.java          |   53 +-
 ...nowflakeStreamingIngestChannelFactory.java |   10 +
 ...owflakeStreamingIngestChannelInternal.java |    8 +-
 ...nowflakeStreamingIngestClientInternal.java |    1 +
 .../streaming/internal/TimestampWrapper.java  |  131 +-
 .../streaming/internal/ArrowBufferTest.java   |    3 +-
 .../streaming/internal/ChannelCacheTest.java  |   20 +-
 .../internal/DataValidationUtilTest.java      |  405 ++--
 .../streaming/internal/FlushServiceTest.java  |   22 +-
 .../internal/ParquetValueParserTest.java      |   58 +-
 .../streaming/internal/RowBufferTest.java     |    3 +
 .../SnowflakeStreamingIngestChannelTest.java  |   29 +-
 .../SnowflakeStreamingIngestClientTest.java   |   18 +-
 .../streaming/internal/StreamingIngestIT.java |   22 +-
 .../datatypes/AbstractDataTypeTest.java       |   23 +-
 .../internal/datatypes/DateTimeIT.java        | 1851 +++++------------
 22 files changed, 1407 insertions(+), 1897 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java b/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java
index 0370771e1..c72629e94 100644
--- a/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java
+++ b/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java
@@ -4,6 +4,7 @@
 
 package net.snowflake.ingest.streaming;
 
+import java.time.ZoneId;
 import net.snowflake.ingest.utils.Utils;
 
 /** A class that is used to open/create a {@link SnowflakeStreamingIngestChannel} */
@@ -13,6 +14,12 @@ public enum OnErrorOption {
     ABORT, // ABORT the entire batch, and throw an exception when we hit the first error
   }
 
+  /**
+   * Default value of the timezone, which will be used for TIMESTAMP_LTZ and TIMESTAMP_TZ column
+   * types when the user input does not have any timezone information.
+   */
+  private static final ZoneId DEFAULT_DEFAULT_TIMEZONE = ZoneId.of("America/Los_Angeles");
+
   // Name of the channel
   private final String channelName;
 
@@ -28,6 +35,9 @@ public enum OnErrorOption {
   // On_error option on this channel
   private final OnErrorOption onErrorOption;
 
+  // Default timezone for TIMESTAMP_LTZ and TIMESTAMP_TZ columns
+  private final ZoneId defaultTimezone;
+
   public static OpenChannelRequestBuilder builder(String channelName) {
     return new OpenChannelRequestBuilder(channelName);
   }
@@ -39,9 +49,11 @@ public static class OpenChannelRequestBuilder {
     private String schemaName;
     private String tableName;
     private OnErrorOption onErrorOption;
+    private ZoneId defaultTimezone;
 
     public OpenChannelRequestBuilder(String channelName) {
       this.channelName = channelName;
+      this.defaultTimezone = DEFAULT_DEFAULT_TIMEZONE;
     }
 
     public OpenChannelRequestBuilder setDBName(String dbName) {
@@ -64,6 +76,11 @@ public OpenChannelRequestBuilder setOnErrorOption(OnErrorOption onErrorOption) {
       return this;
     }
 
+    public OpenChannelRequestBuilder setDefaultTimezone(ZoneId defaultTimezone) {
+      this.defaultTimezone = defaultTimezone;
+      return this;
+    }
+
     public OpenChannelRequest build() {
       return new OpenChannelRequest(this);
     }
@@ -75,12 +92,14 @@ private OpenChannelRequest(OpenChannelRequestBuilder builder) {
     Utils.assertStringNotNullOrEmpty("schema name", builder.schemaName);
     Utils.assertStringNotNullOrEmpty("table name", builder.tableName);
     Utils.assertNotNull("on_error option", builder.onErrorOption);
+    Utils.assertNotNull("default_timezone", builder.defaultTimezone);
 
     this.channelName = builder.channelName;
     this.dbName = builder.dbName;
     this.schemaName = builder.schemaName;
     this.tableName = builder.tableName;
     this.onErrorOption = builder.onErrorOption;
+    this.defaultTimezone = builder.defaultTimezone;
   }
 
   public String getDBName() {
@@ -99,6 +118,10 @@ public String getChannelName() {
     return this.channelName;
   }
 
+  public ZoneId getDefaultTimezone() {
+    return this.defaultTimezone;
+  }
+
   public String getFullyQualifiedTableName() {
     return String.format("%s.%s.%s", this.dbName, this.schemaName, this.tableName);
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
index 71358f405..469a7a72c 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
@@ -4,6 +4,7 @@
 
 package net.snowflake.ingest.streaming;
 
+import java.time.ZoneId;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import javax.annotation.Nullable;
@@ -75,7 +76,149 @@ public interface SnowflakeStreamingIngestChannel {
 
   /**
    * Insert one row into the channel, the row is represented using Map where the key is column name
-   * and the value is a row of data
+   * and the value is a row of data. The following table summarizes supported value types and their
+   * formats:
+   *
+   * <table>
+   *     <tr>
+   *     <th>Snowflake Column Type</th>
+   *     <th>Allowed Java Data Type</th>
+   *     </tr>
+   *     <tr>
+   *         <td>CHAR, VARCHAR</td>
+   *         <td>
+   *             <ul>
+   *                 <li>String</li>
+   *                 <li>primitive data types (int, boolean, char, …)</li>
+   *             </ul>
+   *         </td>
+   *     </tr>
+   *     <tr>
+   *         <td>BINARY</td>
+   *         <td>
+   *             <ul>
+   *                 <li>byte[]</li>
+   *                 <li>String (hex-encoded)</li>
+   *             </ul>
+   *         </td>
+   *     </tr>
+   *     <tr>
+   *         <td>NUMBER, FLOAT</td>
+   *         <td>
+   *             <ul>
+   *                 <li>numeric types (BigInteger, BigDecimal, byte, int, double, …)</li>
+   *                 <li>String</li>
+   *             </ul>
+   *         </td>
+   *     </tr>
+   *     <tr>
+   *         <td>BOOLEAN</td>
+   *         <td>
+   *             <ul>
+   *                 <li>boolean</li>
+   *                 <li>numeric types (BigInteger, BigDecimal, byte, int, double, …)</li>
+   *                 <li>String</li>
+   *             </ul>
+   *             See <a href="https://docs.snowflake.com/en/sql-reference/data-types-logical.html#label-boolean-conversion">boolean conversion details.</a>
+   *         </td>
+   *     </tr>
+   *     <tr>
+   *         <td>TIME</td>
+   *         <td>
+   *             <ul>
+   *                 <li>{@link java.time.LocalTime}</li>
+   *                 <li>{@link java.time.OffsetTime}</li>
+   *                 <li>
+   *                     String (in one of the following formats):
+   *                     <ul>
+   *                         <li>{@link java.time.format.DateTimeFormatter#ISO_LOCAL_TIME}</li>
+   *                         <li>{@link java.time.format.DateTimeFormatter#ISO_OFFSET_TIME}</li>
+   *                         <li>Integer-stored time (see <a href="https://docs.snowflake.com/en/user-guide/date-time-input-output.html#auto-detection-of-integer-stored-date-time-and-timestamp-values">Snowflake Docs</a> for more details)</li>
+   *                     </ul>
+   *                 </li>
+   *
+   *             </ul>
+   *         </td>
+   *     </tr>
+   *     <tr>
+   *         <td>DATE</td>
+   *         <td>
+   *             <ul>
+   *                 <li>{@link java.time.LocalDate}</li>
+   *                 <li>{@link java.time.LocalDateTime}</li>
+   *                 <li>{@link java.time.OffsetDateTime}</li>
+   *                 <li>{@link java.time.ZonedDateTime}</li>
+   *                 <li>{@link java.time.Instant}</li>
+   *                 <li>
+   *                     String (in one of the following formats):
+   *                     <ul>
+   *                         <li>{@link java.time.format.DateTimeFormatter#ISO_LOCAL_DATE}</li>
+   *                         <li>{@link java.time.format.DateTimeFormatter#ISO_LOCAL_DATE_TIME}</li>
+   *                         <li>{@link java.time.format.DateTimeFormatter#ISO_OFFSET_DATE_TIME}</li>
+   *                         <li>{@link java.time.format.DateTimeFormatter#ISO_ZONED_DATE_TIME}</li>
+   *                         <li>Integer-stored date (see <a href="https://docs.snowflake.com/en/user-guide/date-time-input-output.html#auto-detection-of-integer-stored-date-time-and-timestamp-values">Snowflake Docs</a> for more details)</li>
+   *                     </ul>
+   *                 </li>
+   *             </ul>
+   *         </td>
+   *     </tr>
+   *     <tr>
+   *         <td>TIMESTAMP_NTZ, TIMESTAMP_LTZ, TIMESTAMP_TZ</td>
+   *         <td>
+   *             <ul>
+   *                 <li>{@link java.time.LocalDate}</li>
+   *                 <li>{@link java.time.LocalDateTime}</li>
+   *                 <li>{@link java.time.OffsetDateTime}</li>
+   *                 <li>{@link java.time.ZonedDateTime}</li>
+   *                 <li>{@link java.time.Instant}</li>
+   *                 <li>
+   *                     String (in one of the following formats):
+   *                     <ul>
+   *                         <li>{@link java.time.format.DateTimeFormatter#ISO_LOCAL_DATE}</li>
+   *                         <li>{@link java.time.format.DateTimeFormatter#ISO_LOCAL_DATE_TIME}</li>
+   *                         <li>{@link java.time.format.DateTimeFormatter#ISO_OFFSET_DATE_TIME}</li>
+   *                         <li>{@link java.time.format.DateTimeFormatter#ISO_ZONED_DATE_TIME}</li>
+   *                         <li>Integer-stored timestamp (see <a href="https://docs.snowflake.com/en/user-guide/date-time-input-output.html#auto-detection-of-integer-stored-date-time-and-timestamp-values">Snowflake Docs</a> for more details)</li>
+   *                     </ul>
+   *                 </li>
+   *
+   *             </ul>
+   *
+   *             For TIMESTAMP_LTZ and TIMESTAMP_TZ, all input without timezone will be by default interpreted in the timezone "America/Los_Angeles". This can be changed by calling {@link net.snowflake.ingest.streaming.OpenChannelRequest.OpenChannelRequestBuilder#setDefaultTimezone(ZoneId)}.
+   *         </td>
+   *         <tr>
+   *             <td>VARIANT, ARRAY</td>
+   *             <td>
+   *                 <ul>
+   *                     <li>String (must be a valid JSON value)</li>
+   *                     <li>primitive data types and their arrays</li>
+   *                     <li>BigInteger, BigDecimal</li>
+   *                     <li>{@link java.time.LocalDate}</li>
+   *                     <li>{@link java.time.LocalDateTime}</li>
+   *                     <li>{@link java.time.OffsetDateTime}</li>
+   *                     <li>{@link java.time.ZonedDateTime}</li>
+   *                     <li>Map&lt;String, T&gt; where T is a valid VARIANT type</li>
+   *                     <li>T[] where T is a valid VARIANT type</li>
+   *                     <li>List&lt;T&gt; where T is a valid VARIANT type</li>
+   *                 </ul>
+   *             </td>
+   *         </tr>
+   *         <tr>
+   *             <td>OBJECT</td>
+   *             <td>
+   *                 <ul>
+   *                     <li>String (must be a valid JSON object)</li>
+   *                     <li>Map&lt;String, T&gt; where T is a valid variant type</li>
+   *                 </ul>
+   *             </td>
+   *         </tr>
+   *         <tr>
+   *            <td>GEOGRAPHY, GEOMETRY</td>
+   *            <td>Not supported</td>
+   *         </tr>
+   *     </tr>
+   *
+   * </table>
    *
    * @param row object data to write
    * @param offsetToken offset of given row, used for replay in case of failures. It could be null
@@ -86,7 +229,9 @@ public interface SnowflakeStreamingIngestChannel {
 
   /**
    * Insert a batch of rows into the channel, each row is represented using Map where the key is
-   * column name and the value is a row of data
+   * column name and the value is a row of data. See {@link
+   * SnowflakeStreamingIngestChannel#insertRow(Map, String)} for more information about accepted
+   * values.
    *
    * @param rows object data to write
    * @param offsetToken offset of last row in the row-set, used for replay in case of failures, It
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 1e97b3593..c7fa76886 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -4,6 +4,7 @@
 
 package net.snowflake.ingest.streaming.internal;
 
+import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -169,13 +170,17 @@ public int getOrdinal() {
   // ON_ERROR option for this channel
   final OpenChannelRequest.OnErrorOption onErrorOption;
 
+  final ZoneId defaultTimezone;
+
   AbstractRowBuffer(
       OpenChannelRequest.OnErrorOption onErrorOption,
+      ZoneId defaultTimezone,
       BufferAllocator allocator,
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelRuntimeState) {
     this.onErrorOption = onErrorOption;
+    this.defaultTimezone = defaultTimezone;
     this.rowSizeMetric = rowSizeMetric;
     this.channelState = channelRuntimeState;
     this.channelFullyQualifiedName = fullyQualifiedChannelName;
@@ -542,6 +547,7 @@ static EpInfo buildEpInfoFromStats(long rowCount, Map<String, RowBufferStats> co
   /** Row buffer factory. */
   static <T> AbstractRowBuffer<T> createRowBuffer(
       OpenChannelRequest.OnErrorOption onErrorOption,
+      ZoneId defaultTimezone,
       BufferAllocator allocator,
       Constants.BdecVersion bdecVersion,
       String fullyQualifiedChannelName,
@@ -555,6 +561,7 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
         return (AbstractRowBuffer<T>)
             new ArrowRowBuffer(
                 onErrorOption,
+                defaultTimezone,
                 allocator,
                 fullyQualifiedChannelName,
                 rowSizeMetric,
@@ -564,6 +571,7 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
         return (AbstractRowBuffer<T>)
             new ParquetRowBuffer(
                 onErrorOption,
+                defaultTimezone,
                 allocator,
                 fullyQualifiedChannelName,
                 rowSizeMetric,
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 4a38c5d93..9e8b2880e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -8,6 +8,7 @@
 import java.math.BigInteger;
 import java.math.RoundingMode;
 import java.nio.charset.StandardCharsets;
+import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -81,11 +82,18 @@ class ArrowRowBuffer extends AbstractRowBuffer<VectorSchemaRoot> {
   /** Construct a ArrowRowBuffer object. */
   ArrowRowBuffer(
       OpenChannelRequest.OnErrorOption onErrorOption,
+      ZoneId defaultTimezone,
       BufferAllocator allocator,
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelState) {
-    super(onErrorOption, allocator, fullyQualifiedChannelName, rowSizeMetric, channelState);
+    super(
+        onErrorOption,
+        defaultTimezone,
+        allocator,
+        fullyQualifiedChannelName,
+        rowSizeMetric,
+        channelState);
     this.fields = new HashMap<>();
   }
 
@@ -527,7 +535,7 @@ private float convertRowToArrow(
             }
           case TIMESTAMP_LTZ:
           case TIMESTAMP_NTZ:
-            boolean ignoreTimezone = logicalType == ColumnLogicalType.TIMESTAMP_NTZ;
+            boolean trimTimezone = logicalType == ColumnLogicalType.TIMESTAMP_NTZ;
 
             switch (physicalType) {
               case SB8:
@@ -538,9 +546,11 @@ private float convertRowToArrow(
                           stats.getColumnDisplayName(),
                           value,
                           getColumnScale(field.getMetadata()),
-                          ignoreTimezone);
-                  bigIntVector.setSafe(curRowIndex, timestampWrapper.getTimeInScale().longValue());
-                  stats.addIntValue(timestampWrapper.getTimeInScale());
+                          defaultTimezone,
+                          trimTimezone);
+                  BigInteger timestampBinary = timestampWrapper.toBinary(false);
+                  bigIntVector.setSafe(curRowIndex, timestampBinary.longValue());
+                  stats.addIntValue(timestampBinary);
                   rowBufferSize += 8;
                   break;
                 }
@@ -559,11 +569,12 @@ private float convertRowToArrow(
                           stats.getColumnDisplayName(),
                           value,
                           getColumnScale(field.getMetadata()),
-                          ignoreTimezone);
+                          defaultTimezone,
+                          trimTimezone);
                   epochVector.setSafe(curRowIndex, timestampWrapper.getEpoch());
                   fractionVector.setSafe(curRowIndex, timestampWrapper.getFraction());
                   rowBufferSize += 12;
-                  stats.addIntValue(timestampWrapper.getTimeInScale());
+                  stats.addIntValue(timestampWrapper.toBinary(false));
                   break;
                 }
               default:
@@ -587,30 +598,13 @@ private float convertRowToArrow(
                           stats.getColumnDisplayName(),
                           value,
                           getColumnScale(field.getMetadata()),
+                          defaultTimezone,
                           false);
-                  epochVector.setSafe(curRowIndex, timestampWrapper.getTimeInScale().longValue());
-                  timezoneVector.setSafe(
-                      curRowIndex,
-                      timestampWrapper
-                          .getTimeZoneIndex()
-                          .orElseThrow(
-                              () ->
-                                  new SFException(
-                                      ErrorCode.INVALID_ROW,
-                                      value,
-                                      "Unable to parse timezone for TIMESTAMP_TZ column")));
+                  epochVector.setSafe(
+                      curRowIndex, timestampWrapper.toBinary(false).longValueExact());
+                  timezoneVector.setSafe(curRowIndex, timestampWrapper.getTimeZoneIndex());
                   rowBufferSize += 12;
-                  BigInteger timeInBinary =
-                      timestampWrapper
-                          .getSfTimestamp()
-                          .orElseThrow(
-                              () ->
-                                  new SFException(
-                                      ErrorCode.INVALID_ROW,
-                                      value,
-                                      "Unable to parse timestamp for TIMESTAMP_TZ column"))
-                          .toBinary(Integer.parseInt(field.getMetadata().get(COLUMN_SCALE)), true);
-                  stats.addIntValue(timeInBinary);
+                  stats.addIntValue(timestampWrapper.toBinary(true));
                   break;
                 }
               case SB16:
@@ -630,30 +624,13 @@ private float convertRowToArrow(
                           stats.getColumnDisplayName(),
                           value,
                           getColumnScale(field.getMetadata()),
+                          defaultTimezone,
                           false);
                   epochVector.setSafe(curRowIndex, timestampWrapper.getEpoch());
                   fractionVector.setSafe(curRowIndex, timestampWrapper.getFraction());
-                  timezoneVector.setSafe(
-                      curRowIndex,
-                      timestampWrapper
-                          .getTimeZoneIndex()
-                          .orElseThrow(
-                              () ->
-                                  new SFException(
-                                      ErrorCode.INVALID_ROW,
-                                      value,
-                                      "Unable to parse timezone for TIMESTAMP_TZ column")));
+                  timezoneVector.setSafe(curRowIndex, timestampWrapper.getTimeZoneIndex());
                   rowBufferSize += 16;
-                  BigInteger timeInBinary =
-                      timestampWrapper
-                          .getSfTimestamp()
-                          .orElseThrow(
-                              () ->
-                                  new SFException(
-                                      ErrorCode.INVALID_ROW,
-                                      value,
-                                      "Unable to parse timestamp for TIMESTAMP_TZ column"))
-                          .toBinary(Integer.parseInt(field.getMetadata().get(COLUMN_SCALE)), true);
+                  BigInteger timeInBinary = timestampWrapper.toBinary(true);
                   stats.addIntValue(timeInBinary);
                   break;
                 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 0591440f6..92ec86afc 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -4,8 +4,6 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.client.jdbc.internal.snowflake.common.core.SnowflakeDateTimeFormat.DATE;
-import static net.snowflake.client.jdbc.internal.snowflake.common.core.SnowflakeDateTimeFormat.TIMESTAMP;
 import static net.snowflake.ingest.streaming.internal.BinaryStringUtils.unicodeCharactersCount;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -26,19 +24,18 @@
 import java.time.LocalTime;
 import java.time.OffsetDateTime;
 import java.time.OffsetTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
+import java.time.format.DateTimeParseException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.SimpleTimeZone;
-import java.util.TimeZone;
-import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
 import javax.xml.bind.DatatypeConverter;
 import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
-import net.snowflake.client.jdbc.internal.snowflake.common.core.SFDate;
-import net.snowflake.client.jdbc.internal.snowflake.common.core.SFTimestamp;
 import net.snowflake.client.jdbc.internal.snowflake.common.core.SnowflakeDateTimeFormat;
 import net.snowflake.client.jdbc.internal.snowflake.common.util.Power10;
 import net.snowflake.ingest.streaming.internal.serialization.ByteArraySerializer;
@@ -48,6 +45,25 @@
 
 /** Utility class for parsing and validating inputs based on Snowflake types */
 class DataValidationUtil {
+
+  /**
+   * Seconds limit used for integer-stored timestamp scale guessing. Value needs to be aligned with
+   * the value from {@link SnowflakeDateTimeFormat#parse}
+   */
+  private static final long SECONDS_LIMIT_FOR_EPOCH = 31536000000L;
+
+  /**
+   * Milliseconds limit used for integer-stored timestamp scale guessing. Value needs to be aligned
+   * with the value from {@link SnowflakeDateTimeFormat#parse}
+   */
+  private static final long MILLISECONDS_LIMIT_FOR_EPOCH = SECONDS_LIMIT_FOR_EPOCH * 1000L;
+
+  /**
+   * Microseconds limit used for integer-stored timestamp scale guessing. Value needs to be aligned
+   * with the value from {@link SnowflakeDateTimeFormat#parse}
+   */
+  private static final long MICROSECONDS_LIMIT_FOR_EPOCH = SECONDS_LIMIT_FOR_EPOCH * 1000000L;
+
   public static final int BYTES_8_MB = 8 * 1024 * 1024;
   public static final int BYTES_16_MB = 2 * BYTES_8_MB;
 
@@ -55,10 +71,6 @@ class DataValidationUtil {
   // server-side representation. Validation leaves a small buffer for this difference.
   static final int MAX_SEMI_STRUCTURED_LENGTH = BYTES_16_MB - 64;
 
-  private static final TimeZone DEFAULT_TIMEZONE =
-      TimeZone.getTimeZone("America/Los_Angeles"); // default value of TIMEZONE system parameter
-  private static final TimeZone GMT = TimeZone.getTimeZone("GMT");
-
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
   // The version of Jackson we are using does not support serialization of date objects from the
@@ -310,6 +322,103 @@ static String validateAndParseObject(String columnName, Object input) {
     return output;
   }
 
+  /**
+   * Converts user input to offset date time, which is the canonical representation of dates and
+   * timestamps.
+   */
+  private static OffsetDateTime inputToOffsetDateTime(
+      String columnName, String typeName, Object input, ZoneId defaultTimezone) {
+    if (input instanceof OffsetDateTime) {
+      return (OffsetDateTime) input;
+    }
+
+    if (input instanceof ZonedDateTime) {
+      return ((ZonedDateTime) input).toOffsetDateTime();
+    }
+
+    if (input instanceof LocalDateTime) {
+      return ((LocalDateTime) input).atZone(defaultTimezone).toOffsetDateTime();
+    }
+
+    if (input instanceof LocalDate) {
+      return ((LocalDate) input).atStartOfDay().atZone(defaultTimezone).toOffsetDateTime();
+    }
+
+    if (input instanceof Instant) {
+      // Just like integer-stored timestamps, instants are always interpreted in UTC
+      return ((Instant) input).atZone(ZoneOffset.UTC).toOffsetDateTime();
+    }
+
+    if (input instanceof String) {
+      String stringInput = (String) input;
+      {
+        // First, try to parse ZonedDateTime
+        ZonedDateTime zoned = catchParsingError(() -> ZonedDateTime.parse(stringInput));
+        if (zoned != null) {
+          return zoned.toOffsetDateTime();
+        }
+      }
+
+      {
+        // Next, try to parse OffsetDateTime
+        OffsetDateTime offset = catchParsingError(() -> OffsetDateTime.parse(stringInput));
+        if (offset != null) {
+          return offset;
+        }
+      }
+
+      {
+        // Alternatively, try to parse LocalDateTime
+        LocalDateTime localDateTime = catchParsingError(() -> LocalDateTime.parse(stringInput));
+        if (localDateTime != null) {
+          return localDateTime.atZone(defaultTimezone).toOffsetDateTime();
+        }
+      }
+
+      {
+        // Alternatively, try to parse LocalDate
+        LocalDate localDate = catchParsingError(() -> LocalDate.parse(stringInput));
+        if (localDate != null) {
+          return localDate.atStartOfDay().atZone(defaultTimezone).toOffsetDateTime();
+        }
+      }
+
+      {
+        // Alternatively, try to parse integer-stored timestamp
+        // Just like in Snowflake, integer-stored timestamps are always in UTC
+        Instant instant = catchParsingError(() -> parseInstantGuessScale(stringInput));
+        if (instant != null) {
+          return instant.atOffset(ZoneOffset.UTC);
+        }
+      }
+
+      // Couldn't parse anything, throw an exception
+      // TODO Change URL when out of private preview
+      throw valueFormatNotAllowedException(
+          columnName,
+          input.toString(),
+          typeName,
+          "Not a valid value, see"
+              + " https://docs.snowflake.com/en/LIMITEDACCESS/snowpipe-streaming.html"
+              + " for the list of supported formats");
+    }
+
+    // Type is not supported, throw an exception
+    throw typeNotAllowedException(
+        columnName,
+        input.getClass(),
+        typeName,
+        new String[] {"String", "LocalDate", "LocalDateTime", "ZonedDateTime", "OffsetDateTime"});
+  }
+
+  private static <T> T catchParsingError(Supplier<T> op) {
+    try {
+      return op.get();
+    } catch (DateTimeParseException | NumberFormatException e) {
+      return null;
+    }
+  }
+
   /**
    * Validates and parses input for TIMESTAMP_NTZ, TIMESTAMP_LTZ and TIMEATAMP_TZ Snowflake types.
    * Allowed Java types:
@@ -322,69 +431,25 @@ static String validateAndParseObject(String columnName, Object input) {
    *   <li>ZonedDateTime
    * </ul>
    *
+   * @param columnName Column name, used in validation error messages
    * @param input String date in valid format, seconds past the epoch or java.time.* object. Accepts
    *     fractional seconds with precision up to the column's scale
    * @param scale decimal scale of timestamp 16 byte integer
-   * @param ignoreTimezone Must be true for TIMESTAMP_NTZ
-   * @return TimestampWrapper with epoch seconds, fractional seconds, and epoch time in the column
-   *     scale
+   * @param defaultTimezone Input, which does not carry timezone information is going to be
+   *     interpreted in the default timezone.
+   * @param trimTimezone Whether timezone information should be removed from the resulting date,
+   *     should be true for TIMESTAMP_NTZ columns.
+   * @return TimestampWrapper
    */
   static TimestampWrapper validateAndParseTimestamp(
-      String columnName, Object input, int scale, boolean ignoreTimezone) {
-    TimeZone effectiveTimeZone = ignoreTimezone ? GMT : DEFAULT_TIMEZONE;
-    SFTimestamp timestamp;
-    if (input instanceof String) {
-      SnowflakeDateTimeFormat snowflakeDateTimeFormatter = createDateTimeFormatter();
-      timestamp =
-          snowflakeDateTimeFormatter.parse(
-              (String) input, effectiveTimeZone, 0, DATE | TIMESTAMP, ignoreTimezone, null);
-    } else if (input instanceof LocalDate) {
-      timestamp = timeStampFromLocalDate((LocalDate) input, effectiveTimeZone);
-    } else if (input instanceof LocalDateTime) {
-      timestamp = timeStampFromLocalDateTime((LocalDateTime) input, effectiveTimeZone);
-    } else if (input instanceof ZonedDateTime) {
-      timestamp = timestampFromZonedDateTime((ZonedDateTime) input, ignoreTimezone);
-    } else if (input instanceof OffsetDateTime) {
-      timestamp = timestampFromOffsetDateTime((OffsetDateTime) input, ignoreTimezone);
-    } else {
-      throw typeNotAllowedException(
-          columnName,
-          input.getClass(),
-          "TIMESTAMP",
-          new String[] {"String", "LocalDate", "LocalDateTime", "ZonedDateTime", "OffsetDateTime"});
-    }
+      String columnName, Object input, int scale, ZoneId defaultTimezone, boolean trimTimezone) {
+    OffsetDateTime offsetDateTime =
+        inputToOffsetDateTime(columnName, "TIMESTAMP", input, defaultTimezone);
 
-    if (timestamp != null) {
-      long epoch = timestamp.getSeconds().longValue();
-      int fractionInScale = getFractionFromTimestamp(timestamp) / Power10.intTable[9 - scale];
-      BigInteger timeInScale =
-          BigInteger.valueOf(epoch)
-              .multiply(Power10.sb16Table[scale])
-              .add(BigInteger.valueOf(fractionInScale));
-      return new TimestampWrapper(
-          epoch, fractionInScale * Power10.intTable[9 - scale], timeInScale, timestamp);
+    if (trimTimezone) {
+      offsetDateTime = offsetDateTime.withOffsetSameLocal(ZoneOffset.UTC);
     }
-
-    throw valueFormatNotAllowedException(
-        columnName,
-        input.toString(),
-        "TIMESTAMP",
-        "Not a valid timestamp, see"
-            + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
-            + " for the list of supported formats");
-  }
-
-  /**
-   * Given a SFTimestamp, get the number of nanoseconds since the last whole second. This
-   * corresponds to the fraction component for Timestamp types
-   *
-   * @param input SFTimestamp
-   * @return Timestamp fraction value, the number of nanoseconds since the last whole second
-   */
-  private static int getFractionFromTimestamp(SFTimestamp input) {
-    BigDecimal epochSecondsInNanoSeconds =
-        new BigDecimal(input.getSeconds().multiply(BigInteger.valueOf(Power10.intTable[9])));
-    return input.getNanosSinceEpoch().subtract(epochSecondsInNanoSeconds).intValue();
+    return new TimestampWrapper(offsetDateTime, scale);
   }
 
   /**
@@ -494,43 +559,17 @@ static BigDecimal validateAndParseBigDecimal(String columnName, Object input) {
    *
    * <ul>
    *   <li>String
-   *   <li>LocalDate
-   *   <li>LocalDateTime
-   *   <li>OffsetDateTime
-   *   <li>ZonedDateTime
+   *   <li>{@link LocalDate}
+   *   <li>{@link LocalDateTime}
+   *   <li>{@link OffsetDateTime}
+   *   <li>{@link ZonedDateTime}
+   *   <li>{@link Instant}
    * </ul>
    */
   static int validateAndParseDate(String columnName, Object input) {
-    SFTimestamp timestamp;
-    if (input instanceof String) {
-      timestamp =
-          createDateTimeFormatter().parse((String) input, GMT, 0, DATE | TIMESTAMP, true, null);
-    } else if (input instanceof LocalDate) {
-      timestamp = timeStampFromLocalDate((LocalDate) input, GMT);
-    } else if (input instanceof LocalDateTime) {
-      timestamp = timeStampFromLocalDateTime((LocalDateTime) input, GMT);
-    } else if (input instanceof ZonedDateTime) {
-      timestamp = timestampFromZonedDateTime((ZonedDateTime) input, true);
-    } else if (input instanceof OffsetDateTime) {
-      timestamp = timestampFromOffsetDateTime((OffsetDateTime) input, true);
-    } else {
-      throw typeNotAllowedException(
-          columnName,
-          input.getClass(),
-          "DATE",
-          new String[] {"String", "LocalDate", "LocalDateTime", "ZonedDateTime", "OffsetDateTime"});
-    }
-
-    if (timestamp == null)
-      throw valueFormatNotAllowedException(
-          columnName,
-          input,
-          "DATE",
-          "Not a valid date, see"
-              + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#date-formats"
-              + " for the list of supported formats");
-
-    return (int) TimeUnit.MILLISECONDS.toDays(SFDate.fromTimestamp(timestamp).getTime());
+    OffsetDateTime offsetDateTime =
+        inputToOffsetDateTime(columnName, "DATE", input, ZoneOffset.UTC);
+    return Math.toIntExact(offsetDateTime.toLocalDate().toEpochDay());
   }
 
   /**
@@ -579,44 +618,84 @@ static byte[] validateAndParseBinary(
    *
    * <ul>
    *   <li>String
-   *   <li>LocalTime
-   *   <li>OffsetTime
+   *   <li>{@link LocalTime}
+   *   <li>{@link OffsetTime}
    * </ul>
    */
   static BigInteger validateAndParseTime(String columnName, Object input, int scale) {
-    SFTimestamp timestamp;
-    if (input instanceof String) {
-      String stringInput = (String) input;
-      timestamp =
-          createDateTimeFormatter()
-              .parse(stringInput, GMT, 0, SnowflakeDateTimeFormat.TIME, true, null);
-    } else if (input instanceof LocalTime) {
-      timestamp =
-          timeStampFromLocalDateTime(((LocalTime) input).atDate(LocalDate.ofEpochDay(0)), GMT);
+    if (input instanceof LocalTime) {
+      LocalTime localTime = (LocalTime) input;
+      return BigInteger.valueOf(localTime.toNanoOfDay()).divide(Power10.sb16Table[9 - scale]);
     } else if (input instanceof OffsetTime) {
-      timestamp =
-          timeStampFromLocalDateTime(
-              ((OffsetTime) input).toLocalTime().atDate(LocalDate.ofEpochDay(0)), GMT);
-    } else {
-      throw typeNotAllowedException(
-          columnName, input.getClass(), "TIME", new String[] {"String", "LocalTime", "OffsetTime"});
-    }
+      return validateAndParseTime(columnName, ((OffsetTime) input).toLocalTime(), scale);
+    } else if (input instanceof String) {
+      String stringInput = (String) input;
+      {
+        // First, try to parse LocalTime
+        LocalTime localTime = catchParsingError(() -> LocalTime.parse(stringInput));
+        if (localTime != null) {
+          return validateAndParseTime(columnName, localTime, scale);
+        }
+      }
 
-    if (timestamp == null) {
+      {
+        // Alternatively, try to parse OffsetTime
+        OffsetTime offsetTime = catchParsingError((() -> OffsetTime.parse(stringInput)));
+        if (offsetTime != null) {
+          return validateAndParseTime(columnName, offsetTime.toLocalTime(), scale);
+        }
+      }
+
+      {
+        // Alternatively, try to parse integer-stored time
+        Instant parsedInstant = catchParsingError(() -> parseInstantGuessScale(stringInput));
+        if (parsedInstant != null) {
+          return validateAndParseTime(
+              columnName,
+              LocalDateTime.ofInstant(parsedInstant, ZoneOffset.UTC).toLocalTime(),
+              scale);
+        }
+      }
+
+      // TODO Change URL when out of private preview
       throw valueFormatNotAllowedException(
           columnName,
           input,
           "TIME",
           "Not a valid time, see"
-              + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#time-formats"
+              + " https://docs.snowflake.com/en/LIMITEDACCESS/snowpipe-streaming.html"
               + " for the list of supported formats");
+
+    } else {
+      throw typeNotAllowedException(
+          columnName, input.getClass(), "TIME", new String[] {"String", "LocalTime", "OffsetTime"});
     }
+  }
 
-    return timestamp
-        .getNanosSinceEpoch()
-        .toBigInteger()
-        .divide(Power10.sb16Table[9 - scale])
-        .mod(BigInteger.valueOf(24L * 60 * 60).multiply(Power10.sb16Table[scale]));
+  /**
+   * Attempts to parse integer-stored date from string input. Tries to guess the scale according to
+   * the rules documented at
+   * https://docs.snowflake.com/en/user-guide/date-time-input-output.html#auto-detection-of-integer-stored-date-time-and-timestamp-values.
+   *
+   * @param input String to parse, must represent a valid long
+   * @return Instant representing the input
+   * @throws NumberFormatException If the input in not a valid long
+   */
+  private static Instant parseInstantGuessScale(String input) {
+    long epochNanos;
+    long val = Long.parseLong(input);
+
+    if (val > -SECONDS_LIMIT_FOR_EPOCH && val < SECONDS_LIMIT_FOR_EPOCH) {
+      epochNanos = val * Power10.intTable[9];
+    } else if (val > -MILLISECONDS_LIMIT_FOR_EPOCH && val < MILLISECONDS_LIMIT_FOR_EPOCH) {
+      epochNanos = val * Power10.intTable[6];
+    } else if (val > -MICROSECONDS_LIMIT_FOR_EPOCH && val < MICROSECONDS_LIMIT_FOR_EPOCH) {
+      epochNanos = val * Power10.intTable[3];
+    } else {
+      epochNanos = val;
+    }
+    return Instant.ofEpochSecond(
+        epochNanos / Power10.intTable[9], epochNanos % Power10.intTable[9]);
   }
 
   /**
@@ -764,54 +843,6 @@ private static String sanitizeValueForExceptionMessage(Object value) {
     return valueString.length() <= maxSize ? valueString : valueString.substring(0, 20) + "...";
   }
 
-  /**
-   * Constructs SFTimestamp from {@link LocalDate}. Default timezone is used for _TZ and _LTZ and
-   * UTC for _NTZ.
-   */
-  private static SFTimestamp timeStampFromLocalDate(LocalDate date, TimeZone tz) {
-    return timeStampFromLocalDateTime(date.atStartOfDay(), tz);
-  }
-
-  /**
-   * Constructs SFTimestamp from {@link LocalDateTime}. Default timezone is used for _TZ and _LTZ
-   * and UTC for _NTZ.
-   */
-  private static SFTimestamp timeStampFromLocalDateTime(LocalDateTime localDateTime, TimeZone tz) {
-    return timestampFromInstant(localDateTime.atZone(tz.toZoneId()).toInstant(), tz);
-  }
-
-  /** Constructs SFTimestamp from {@link ZonedDateTime}. Timezone is dropped for _NTZ. */
-  private static SFTimestamp timestampFromZonedDateTime(
-      ZonedDateTime zonedDateTime, boolean ignoreTimezone) {
-    if (ignoreTimezone) {
-      LocalDateTime local = zonedDateTime.toLocalDateTime();
-      return timeStampFromLocalDateTime(local, GMT);
-    }
-
-    TimeZone timeZone = TimeZone.getTimeZone(zonedDateTime.getZone().toString());
-    return timestampFromInstant(zonedDateTime.toInstant(), timeZone);
-  }
-
-  /** Constructs SFTimestamp from {@link OffsetDateTime}. Timezone is dropped for _NTZ. */
-  private static SFTimestamp timestampFromOffsetDateTime(
-      OffsetDateTime offsetDateTime, boolean dropTimezone) {
-    if (dropTimezone) {
-      LocalDateTime local = offsetDateTime.toLocalDateTime();
-      return timeStampFromLocalDateTime(local, GMT);
-    }
-    TimeZone tz =
-        new SimpleTimeZone(
-            offsetDateTime.getOffset().getTotalSeconds() * 1000,
-            "GENERATED_SNOWPIPE_STREAMING:" + offsetDateTime.getOffset());
-    return timestampFromInstant(offsetDateTime.toInstant(), tz);
-  }
-
-  /** Constructs SFTimestamp from {@link Instant} and time zone. */
-  private static SFTimestamp timestampFromInstant(Instant instant, TimeZone timeZone) {
-    return SFTimestamp.fromNanoseconds(
-        instant.getEpochSecond() * Power10.intTable[9] + instant.getNano(), timeZone);
-  }
-
   /**
    * Validates that a string is valid UTF-8 string. It catches situations like unmatched high/low
    * UTF-16 surrogate, for example.
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index f1d0daa9a..46a756b60 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -9,6 +9,7 @@
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
+import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -60,13 +61,20 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
   /** Construct a ParquetRowBuffer object. */
   ParquetRowBuffer(
       OpenChannelRequest.OnErrorOption onErrorOption,
+      ZoneId defaultTimezone,
       BufferAllocator allocator,
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelRuntimeState,
       boolean bufferForTests,
       boolean enableParquetInternalBuffering) {
-    super(onErrorOption, allocator, fullyQualifiedChannelName, rowSizeMetric, channelRuntimeState);
+    super(
+        onErrorOption,
+        defaultTimezone,
+        allocator,
+        fullyQualifiedChannelName,
+        rowSizeMetric,
+        channelRuntimeState);
     fieldIndex = new HashMap<>();
     metadata = new HashMap<>();
     data = new ArrayList<>();
@@ -185,7 +193,8 @@ private float addRow(
       PrimitiveType.PrimitiveTypeName typeName =
           columnDescriptor.getPrimitiveType().getPrimitiveTypeName();
       ParquetValueParser.ParquetBufferValue valueWithSize =
-          ParquetValueParser.parseColumnValueToParquet(value, column, typeName, stats);
+          ParquetValueParser.parseColumnValueToParquet(
+              value, column, typeName, stats, defaultTimezone);
       indexedRow[colIndex] = valueWithSize.getValue();
       size += valueWithSize.getSize();
     }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
index dcf119204..0f0442211 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
@@ -8,6 +8,7 @@
 import java.math.BigInteger;
 import java.math.RoundingMode;
 import java.nio.charset.StandardCharsets;
+import java.time.ZoneId;
 import java.util.Optional;
 import javax.annotation.Nullable;
 import net.snowflake.ingest.utils.ErrorCode;
@@ -49,7 +50,8 @@ static ParquetBufferValue parseColumnValueToParquet(
       Object value,
       ColumnMetadata columnMetadata,
       PrimitiveType.PrimitiveTypeName typeName,
-      RowBufferStats stats) {
+      RowBufferStats stats,
+      ZoneId defaultTimezone) {
     Utils.assertNotNull("Parquet column stats", stats);
     float size = 0F;
     if (value != null) {
@@ -86,7 +88,8 @@ static ParquetBufferValue parseColumnValueToParquet(
                   columnMetadata.getScale(),
                   Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
                   logicalType,
-                  physicalType);
+                  physicalType,
+                  defaultTimezone);
           value = longValue;
           stats.addIntValue(BigInteger.valueOf(longValue));
           size = 8;
@@ -118,7 +121,8 @@ static ParquetBufferValue parseColumnValueToParquet(
                   columnMetadata.getScale(),
                   Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
                   logicalType,
-                  physicalType);
+                  physicalType,
+                  defaultTimezone);
           stats.addIntValue(intRep);
           value = getSb16Bytes(intRep);
           size += 16;
@@ -194,7 +198,8 @@ private static long getInt64Value(
       int scale,
       int precision,
       AbstractRowBuffer.ColumnLogicalType logicalType,
-      AbstractRowBuffer.ColumnPhysicalType physicalType) {
+      AbstractRowBuffer.ColumnPhysicalType physicalType,
+      ZoneId defaultTimezone) {
     long longValue;
     switch (logicalType) {
       case TIME:
@@ -203,23 +208,18 @@ private static long getInt64Value(
         break;
       case TIMESTAMP_LTZ:
       case TIMESTAMP_NTZ:
-        boolean ignoreTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
+        boolean trimTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
         longValue =
-            DataValidationUtil.validateAndParseTimestamp(columnName, value, scale, ignoreTimezone)
-                .getTimeInScale()
+            DataValidationUtil.validateAndParseTimestamp(
+                    columnName, value, scale, defaultTimezone, trimTimezone)
+                .toBinary(false)
                 .longValue();
         break;
       case TIMESTAMP_TZ:
         longValue =
-            DataValidationUtil.validateAndParseTimestamp(columnName, value, scale, false)
-                .getSfTimestamp()
-                .orElseThrow(
-                    () ->
-                        new SFException(
-                            ErrorCode.INVALID_ROW,
-                            value,
-                            "Unable to parse timestamp for TIMESTAMP_TZ column"))
-                .toBinary(scale, true)
+            DataValidationUtil.validateAndParseTimestamp(
+                    columnName, value, scale, defaultTimezone, false)
+                .toBinary(true)
                 .longValue();
         break;
       case FIXED:
@@ -251,24 +251,19 @@ private static BigInteger getSb16Value(
       int scale,
       int precision,
       AbstractRowBuffer.ColumnLogicalType logicalType,
-      AbstractRowBuffer.ColumnPhysicalType physicalType) {
+      AbstractRowBuffer.ColumnPhysicalType physicalType,
+      ZoneId defaultTimezone) {
     switch (logicalType) {
       case TIMESTAMP_TZ:
-        return DataValidationUtil.validateAndParseTimestamp(columnName, value, scale, false)
-            .getSfTimestamp()
-            .orElseThrow(
-                () ->
-                    new SFException(
-                        ErrorCode.INVALID_ROW,
-                        value,
-                        "Unable to parse timestamp for TIMESTAMP_TZ column"))
-            .toBinary(scale, true);
+        return DataValidationUtil.validateAndParseTimestamp(
+                columnName, value, scale, defaultTimezone, false)
+            .toBinary(true);
       case TIMESTAMP_LTZ:
       case TIMESTAMP_NTZ:
-        boolean ignoreTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
+        boolean trimTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
         return DataValidationUtil.validateAndParseTimestamp(
-                columnName, value, scale, ignoreTimezone)
-            .getTimeInScale();
+                columnName, value, scale, defaultTimezone, trimTimezone)
+            .toBinary(false);
       case FIXED:
         BigDecimal bigDecimalValue =
             DataValidationUtil.validateAndParseBigDecimal(columnName, value);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java
index 73184019d..ed75796a4 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java
@@ -4,6 +4,7 @@
 
 package net.snowflake.ingest.streaming.internal;
 
+import java.time.ZoneId;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.utils.Utils;
 import org.apache.arrow.memory.BufferAllocator;
@@ -29,6 +30,8 @@ static class SnowflakeStreamingIngestChannelBuilder<T> {
     private Long encryptionKeyId;
     private OpenChannelRequest.OnErrorOption onErrorOption;
 
+    private ZoneId defaultTimezone;
+
     private SnowflakeStreamingIngestChannelBuilder(String name) {
       this.name = name;
     }
@@ -79,6 +82,11 @@ SnowflakeStreamingIngestChannelBuilder<T> setOnErrorOption(
       return this;
     }
 
+    SnowflakeStreamingIngestChannelBuilder<T> setDefaultTimezone(ZoneId zoneId) {
+      this.defaultTimezone = zoneId;
+      return this;
+    }
+
     SnowflakeStreamingIngestChannelBuilder<T> setOwningClient(
         SnowflakeStreamingIngestClientInternal<T> client) {
       this.owningClient = client;
@@ -96,6 +104,7 @@ SnowflakeStreamingIngestChannelInternal<T> build() {
       Utils.assertStringNotNullOrEmpty("encryption key", this.encryptionKey);
       Utils.assertNotNull("encryption key_id", this.encryptionKeyId);
       Utils.assertNotNull("on_error option", this.onErrorOption);
+      Utils.assertNotNull("default timezone", this.defaultTimezone);
       BufferAllocator allocator = createBufferAllocator();
       return new SnowflakeStreamingIngestChannelInternal<>(
           this.name,
@@ -109,6 +118,7 @@ SnowflakeStreamingIngestChannelInternal<T> build() {
           this.encryptionKey,
           this.encryptionKeyId,
           this.onErrorOption,
+          this.defaultTimezone,
           this.owningClient.getParameterProvider().getBlobFormatVersion(),
           allocator);
     }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 349721ac9..982f3290d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -11,6 +11,8 @@
 import static net.snowflake.ingest.utils.ParameterProvider.MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT;
 
 import com.google.common.annotations.VisibleForTesting;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -76,7 +78,8 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
       SnowflakeStreamingIngestClientInternal<T> client,
       String encryptionKey,
       Long encryptionKeyId,
-      OpenChannelRequest.OnErrorOption onErrorOption) {
+      OpenChannelRequest.OnErrorOption onErrorOption,
+      ZoneOffset defaultTimezone) {
     this(
         name,
         dbName,
@@ -89,6 +92,7 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
         encryptionKey,
         encryptionKeyId,
         onErrorOption,
+        defaultTimezone,
         client.getParameterProvider().getBlobFormatVersion(),
         new RootAllocator());
   }
@@ -106,6 +110,7 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
       String encryptionKey,
       Long encryptionKeyId,
       OpenChannelRequest.OnErrorOption onErrorOption,
+      ZoneId defaultTimezone,
       Constants.BdecVersion bdecVersion,
       BufferAllocator allocator) {
     this.isClosed = false;
@@ -117,6 +122,7 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
     this.rowBuffer =
         AbstractRowBuffer.createRowBuffer(
             onErrorOption,
+            defaultTimezone,
             allocator,
             bdecVersion,
             getFullyQualifiedName(),
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 05ef85322..8e319f408 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -328,6 +328,7 @@ public SnowflakeStreamingIngestChannelInternal<?> openChannel(OpenChannelRequest
               .setEncryptionKey(response.getEncryptionKey())
               .setEncryptionKeyId(response.getEncryptionKeyId())
               .setOnErrorOption(request.getOnErrorOption())
+              .setDefaultTimezone(request.getDefaultTimezone())
               .build();
 
       // Setup the row buffer schema
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/TimestampWrapper.java b/src/main/java/net/snowflake/ingest/streaming/internal/TimestampWrapper.java
index b0c82369d..deb664edd 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/TimestampWrapper.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/TimestampWrapper.java
@@ -1,79 +1,98 @@
 /*
- * Copyright (c) 2021 Snowflake Computing Inc. All rights reserved.
+ * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved.
  */
 
 package net.snowflake.ingest.streaming.internal;
 
+import java.math.BigDecimal;
 import java.math.BigInteger;
-import java.util.Objects;
-import java.util.Optional;
-import net.snowflake.client.jdbc.internal.snowflake.common.core.SFTimestamp;
+import java.math.RoundingMode;
+import java.time.OffsetDateTime;
+import net.snowflake.client.jdbc.internal.snowflake.common.util.Power10;
 
-class TimestampWrapper {
-
-  /** Seconds since the epoch */
-  private long epoch;
-
-  /** Fraction of a second since the epoch in nanoseconds */
-  private int fraction;
-
-  /** Epoch time in column's scale, e.g. for scale=3 this milliseconds past the epoch */
-  private BigInteger timeInScale;
-
-  /** SFTimestamp including timezone information */
-  private Optional<SFTimestamp> sfTimestamp = Optional.empty();
-
-  public TimestampWrapper(long epoch, int fraction, BigInteger timeInScale) {
-    this.epoch = epoch;
-    this.fraction = fraction;
-    this.timeInScale = timeInScale;
+/**
+ * This class represents the outcome of timestamp parsing and validation. It contains methods needed
+ * to serialize timestamps into Arrow and Parquet.
+ */
+public class TimestampWrapper {
+
+  /** Epoch seconds */
+  private final long epoch;
+
+  /** Fractional part of the second */
+  private final int fraction;
+
+  /** Timezone offset in seconds */
+  private final int timezoneOffsetSeconds;
+
+  /** Scale of the timestamp column (0-9) */
+  private final int scale;
+
+  /**
+   * How many bits should be reserver for the timezone part. Needs to be aligned with {@link
+   * net.snowflake.client.jdbc.internal.snowflake.common.core.SFTimestamp#BITS_FOR_TIMEZONE}
+   */
+  private static final int BITS_FOR_TIMEZONE = 14;
+
+  /**
+   * Mask of the timezone bits. Needs to be aligned with {@link
+   * net.snowflake.client.jdbc.internal.snowflake.common.core.SFTimestamp#MASK_OF_TIMEZONE}
+   */
+  private static final int MASK_OF_TIMEZONE = (1 << BITS_FOR_TIMEZONE) - 1;
+
+  /** Create a new instance from {@link OffsetDateTime} and its scale. */
+  public TimestampWrapper(OffsetDateTime offsetDateTime, int scale) {
+    if (scale < 0 || scale > 9) {
+      throw new IllegalArgumentException(
+          String.format("Scale must be between 0 and 9, actual: %d", scale));
+    }
+    this.epoch = offsetDateTime.toEpochSecond();
+    this.fraction =
+        offsetDateTime.getNano() / Power10.intTable[9 - scale] * Power10.intTable[9 - scale];
+    this.timezoneOffsetSeconds = offsetDateTime.getOffset().getTotalSeconds();
+    this.scale = scale;
   }
 
-  public TimestampWrapper(
-      long epoch, int fraction, BigInteger timeInScale, SFTimestamp sfTimestamp) {
-    this.epoch = epoch;
-    this.fraction = fraction;
-    this.timeInScale = timeInScale;
-    this.sfTimestamp = Optional.ofNullable(sfTimestamp);
+  /**
+   * Convert the timestamp to a binary representation. Needs to be aligned with {@link
+   * net.snowflake.client.jdbc.internal.snowflake.common.core.SFTimestamp#toBinary}.
+   */
+  public BigInteger toBinary(boolean includeTimezone) {
+    BigDecimal timeInNs =
+        BigDecimal.valueOf(epoch).scaleByPowerOfTen(9).add(new BigDecimal(fraction));
+    BigDecimal scaledTime = timeInNs.scaleByPowerOfTen(scale - 9);
+    scaledTime = scaledTime.setScale(0, RoundingMode.DOWN);
+    BigInteger fcpInt = scaledTime.unscaledValue();
+    if (includeTimezone) {
+      int offsetMin = timezoneOffsetSeconds / 60;
+      assert offsetMin >= -1440 && offsetMin <= 1440;
+      offsetMin += 1440;
+      fcpInt = fcpInt.shiftLeft(14);
+      fcpInt = fcpInt.add(BigInteger.valueOf(offsetMin & MASK_OF_TIMEZONE));
+    }
+    return fcpInt;
   }
 
+  /** Get epoch in seconds */
   public long getEpoch() {
     return epoch;
   }
 
+  /** Get fractional part of a second */
   public int getFraction() {
     return fraction;
   }
 
-  public BigInteger getTimeInScale() {
-    return timeInScale;
-  }
-
-  public Optional<Integer> getTimezoneOffset() {
-    return sfTimestamp.map(t -> t.getTimeZoneOffsetMillis());
-  }
-
-  public Optional<Integer> getTimeZoneIndex() {
-    return this.getTimezoneOffset().map(t -> (t / 1000 / 60) + 1440);
-  }
-
-  public Optional<SFTimestamp> getSfTimestamp() {
-    return sfTimestamp;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) return true;
-    if (o == null || getClass() != o.getClass()) return false;
-    TimestampWrapper that = (TimestampWrapper) o;
-    return epoch == that.epoch
-        && fraction == that.fraction
-        && Objects.equals(timeInScale, that.timeInScale)
-        && sfTimestamp.equals(that.sfTimestamp);
+  /** Get timezone offset in seconds */
+  public int getTimezoneOffsetSeconds() {
+    return timezoneOffsetSeconds;
   }
 
-  @Override
-  public int hashCode() {
-    return Objects.hash(epoch, fraction, timeInScale, sfTimestamp);
+  /**
+   * Get timezone index, 1440 means UTC. Calculation needs to be aligned with {@link
+   * net.snowflake.client.jdbc.internal.snowflake.common.core.SFTimestamp#toBinary}
+   */
+  public int getTimeZoneIndex() {
+    return timezoneOffsetSeconds / 60 + 1440;
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
index 08e559bcc..f2cc72a16 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
@@ -2,6 +2,7 @@
 
 import static net.snowflake.ingest.streaming.internal.ArrowRowBuffer.DECIMAL_BIT_WIDTH;
 
+import java.time.ZoneOffset;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -29,7 +30,7 @@ public void setupRowBuffer() {
   ArrowRowBuffer createTestBuffer(OpenChannelRequest.OnErrorOption onErrorOption) {
     ChannelRuntimeState initialState = new ChannelRuntimeState("0", 0L, true);
     return new ArrowRowBuffer(
-        onErrorOption, new RootAllocator(), "test.buffer", rs -> {}, initialState);
+        onErrorOption, ZoneOffset.UTC, new RootAllocator(), "test.buffer", rs -> {}, initialState);
   }
 
   @Test
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java
index 7354a0f0d..ea88d618c 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java
@@ -1,5 +1,7 @@
 package net.snowflake.ingest.streaming.internal;
 
+import static java.time.ZoneOffset.UTC;
+
 import java.util.Iterator;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -35,7 +37,8 @@ public void setup() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
     channel2 =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel2",
@@ -48,7 +51,8 @@ public void setup() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
     channel3 =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel3",
@@ -61,7 +65,8 @@ public void setup() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
     cache.addChannel(channel1);
     cache.addChannel(channel2);
     cache.addChannel(channel3);
@@ -86,7 +91,8 @@ public void testAddChannel() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
     cache.addChannel(channel);
     Assert.assertEquals(1, cache.getSize());
     Assert.assertTrue(channel == cache.iterator().next().getValue().get(channelName));
@@ -103,7 +109,8 @@ public void testAddChannel() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
     cache.addChannel(channelDup);
     // The old channel should be invalid now
     Assert.assertTrue(!channel.isValid());
@@ -183,7 +190,8 @@ public void testRemoveChannel() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
     cache.removeChannelIfSequencersMatch(channel3Dup);
     // Verify that remove the same channel with a different channel sequencer is a no op
     Assert.assertEquals(1, cache.getSize());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index eff9788cf..76ab1809f 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -1,5 +1,6 @@
 package net.snowflake.ingest.streaming.internal;
 
+import static java.time.ZoneOffset.UTC;
 import static net.snowflake.ingest.TestUtils.buildString;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.BYTES_16_MB;
 import static net.snowflake.ingest.streaming.internal.DataValidationUtil.BYTES_8_MB;
@@ -25,12 +26,14 @@
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
+import java.time.Instant;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.LocalTime;
 import java.time.OffsetDateTime;
 import java.time.OffsetTime;
 import java.time.ZoneId;
+import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.Arrays;
 import java.util.Collections;
@@ -40,7 +43,6 @@
 import java.util.Map;
 import java.util.Optional;
 import javax.xml.bind.DatatypeConverter;
-import net.snowflake.client.jdbc.internal.snowflake.common.core.SFTimestamp;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 import org.junit.Assert;
@@ -58,6 +60,7 @@ private void expectErrorCodeAndMessage(
       assertEquals(expectedErrorCode.getMessageCode(), e.getVendorCode());
       if (expectedExceptionMessage != null) assertEquals(expectedExceptionMessage, e.getMessage());
     } catch (Exception e) {
+      e.printStackTrace();
       Assert.fail("Invalid error through");
     }
   }
@@ -66,97 +69,128 @@ private void expectError(ErrorCode expectedErrorCode, Runnable action) {
     expectErrorCodeAndMessage(expectedErrorCode, null, action);
   }
 
+  @Test
+  public void testValidateAndParseDate() {
+    assertEquals(9, validateAndParseDate("COL", LocalDate.of(1970, 1, 10)));
+    assertEquals(9, validateAndParseDate("COL", LocalDateTime.of(1970, 1, 10, 1, 0)));
+    assertEquals(
+        9,
+        validateAndParseDate(
+            "COL", OffsetDateTime.of(1970, 1, 10, 1, 0, 34, 123456789, ZoneOffset.of("-07:00"))));
+    assertEquals(
+        9,
+        validateAndParseDate(
+            "COL", OffsetDateTime.of(1970, 1, 10, 1, 0, 34, 123456789, ZoneOffset.of("+07:00"))));
+    assertEquals(
+        9,
+        validateAndParseDate(
+            "COL",
+            ZonedDateTime.of(1970, 1, 10, 1, 0, 34, 123456789, ZoneId.of("America/Los_Angeles"))));
+    assertEquals(
+        9,
+        validateAndParseDate(
+            "COL", ZonedDateTime.of(1970, 1, 10, 1, 0, 34, 123456789, ZoneId.of("Asia/Tokyo"))));
+    assertEquals(19380, validateAndParseDate("COL", Instant.ofEpochMilli(1674478926000L)));
+
+    assertEquals(-923, validateAndParseDate("COL", "1967-06-23"));
+    assertEquals(-923, validateAndParseDate("COL", "1967-06-23T01:01:01"));
+    assertEquals(18464, validateAndParseDate("COL", "2020-07-21"));
+    assertEquals(18464, validateAndParseDate("COL", "2020-07-21T23:31:00"));
+    assertEquals(18464, validateAndParseDate("COL", "2020-07-21T23:31:00+07:00"));
+    assertEquals(18464, validateAndParseDate("COL", "2020-07-21T23:31:00-07:00"));
+    assertEquals(
+        18464, validateAndParseDate("COL", "2020-07-21T23:31:00-07:00[America/Los_Angeles]"));
+    assertEquals(18464, validateAndParseDate("COL", "2020-07-21T23:31:00+09:00[Asia/Tokyo]"));
+
+    // Test integer-stored date
+    assertEquals(19380, validateAndParseDate("COL", "1674478926"));
+    assertEquals(19380, validateAndParseDate("COL", "1674478926000"));
+    assertEquals(19380, validateAndParseDate("COL", "1674478926000000"));
+    assertEquals(19380, validateAndParseDate("COL", "1674478926000000000"));
+
+    // Time input is not supported
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "20:57:01"));
+
+    // Test forbidden values
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", new Object()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", LocalTime.now()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", OffsetTime.now()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", new java.util.Date()));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", ""));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "foo"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "1.0"));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", 'c'));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", 1));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", 1L));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", 1.25));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", BigInteger.valueOf(1)));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", BigDecimal.valueOf(1.25)));
+  }
+
   @Test
   public void testValidateAndParseTime() {
-    assertEquals(5L, validateAndParseTime("COL", "00:00:05", 0).longValueExact());
-    assertEquals(5000L, validateAndParseTime("COL", "00:00:05", 3).longValueExact());
-    assertEquals(5000L, validateAndParseTime("COL", "00:00:05.000", 3).longValueExact());
-    assertEquals(5123L, validateAndParseTime("COL", "00:00:05.123", 3).longValueExact());
-    assertEquals(5123L, validateAndParseTime("COL", "00:00:05.123456", 3).longValueExact());
+    // Test local time
+    assertEquals(46920, validateAndParseTime("COL", "13:02", 0).longValueExact());
+    assertEquals(46926, validateAndParseTime("COL", "13:02:06", 0).longValueExact());
+    assertEquals(469260, validateAndParseTime("COL", "13:02:06", 1).longValueExact());
+    assertEquals(46926000000000L, validateAndParseTime("COL", "13:02:06", 9).longValueExact());
+
+    assertEquals(46926, validateAndParseTime("COL", "13:02:06.1234", 0).longValueExact());
+    assertEquals(469261, validateAndParseTime("COL", "13:02:06.1234", 1).longValueExact());
+    assertEquals(46926123400000L, validateAndParseTime("COL", "13:02:06.1234", 9).longValueExact());
+
+    assertEquals(46926, validateAndParseTime("COL", "13:02:06.123456789", 0).longValueExact());
+    assertEquals(469261, validateAndParseTime("COL", "13:02:06.123456789", 1).longValueExact());
     assertEquals(
-        5123456789L, validateAndParseTime("COL", "00:00:05.123456789", 9).longValueExact());
+        46926123456789L, validateAndParseTime("COL", "13:02:06.123456789", 9).longValueExact());
 
-    assertEquals(72L, validateAndParseTime("COL", "72", 0).longValueExact());
-    assertEquals(72000L, validateAndParseTime("COL", "72", 3).longValueExact());
+    // Test that offset time does not make any difference
+    assertEquals(
+        46926123456789L,
+        validateAndParseTime("COL", "13:02:06.123456789+09:00", 9).longValueExact());
+    assertEquals(
+        46926123456789L,
+        validateAndParseTime("COL", "13:02:06.123456789-09:00", 9).longValueExact());
+
+    // Test integer-stored time and scale guessing
+    assertEquals(46926L, validateAndParseTime("COL", "1674478926", 0).longValueExact());
+    assertEquals(46926L, validateAndParseTime("COL", "1674478926123", 0).longValueExact());
+    assertEquals(46926L, validateAndParseTime("COL", "1674478926123456", 0).longValueExact());
+    assertEquals(46926L, validateAndParseTime("COL", "1674478926123456789", 0).longValueExact());
+
+    assertEquals(469260L, validateAndParseTime("COL", "1674478926", 1).longValueExact());
+    assertEquals(469261L, validateAndParseTime("COL", "1674478926123", 1).longValueExact());
+    assertEquals(469261L, validateAndParseTime("COL", "1674478926123456", 1).longValueExact());
+    assertEquals(469261L, validateAndParseTime("COL", "1674478926123456789", 1).longValueExact());
+
+    assertEquals(46926000000000L, validateAndParseTime("COL", "1674478926", 9).longValueExact());
+    assertEquals(46926123000000L, validateAndParseTime("COL", "1674478926123", 9).longValueExact());
+    assertEquals(
+        46926123456000L, validateAndParseTime("COL", "1674478926123456", 9).longValueExact());
+    assertEquals(
+        46926123456789L, validateAndParseTime("COL", "1674478926123456789", 9).longValueExact());
 
-    // Timestamps are rejected
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2/18/2008 02:36:48", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20:57", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20:57:01", 9));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20:57:01 +07:00", 9));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20:57:01 +0700", 9));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20:57:01-07", 9));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20:57:01-07:00", 9));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20:57:01.123456", 9));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("COL", "2013-04-28 20:57:01.123456789 +07:00", 9));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("COL", "2013-04-28 20:57:01.123456789 +0700", 9));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("COL", "2013-04-28 20:57:01.123456789+07", 9));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("COL", "2013-04-28 20:57:01.123456789+07:00", 9));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28 20:57+07:00", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28T20", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28T20:57", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28T20:57:01", 9));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28T20:57:01-07:00", 9));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28T20:57:01.123456", 9));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("COL", "2013-04-28T20:57:01.123456789+07:00", 9));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28T20:57+07:00", 9));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("COL", "Mon Jul 08 18:09:51 +0000 2013", 9));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("COL", "Thu, 21 Dec 2000 04:01:07 PM", 9));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("COL", "Thu, 21 Dec 2000 04:01:07 PM +0200", 9));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("COL", "Thu, 21 Dec 2000 04:01:07.123456789 PM", 9));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("COL", "Thu, 21 Dec 2000 04:01:07.123456789 PM +0200", 9));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "Thu, 21 Dec 2000 16:01:07", 9));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("COL", "Thu, 21 Dec 2000 16:01:07 +0200", 9));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("COL", "Thu, 21 Dec 2000 16:01:07.123456789", 9));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("COL", "Thu, 21 Dec 2000 16:01:07.123456789 +0200", 9));
+    // Test Java objects
+    assertEquals(
+        46926123456789L,
+        validateAndParseTime("COL", LocalTime.of(13, 2, 6, 123456789), 9).longValueExact());
+    assertEquals(
+        46926123456789L,
+        validateAndParseTime("COL", OffsetTime.of(13, 2, 6, 123456789, ZoneOffset.of("+09:00")), 9)
+            .longValueExact());
 
-    // Dates are rejected
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2013-04-28", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "17-DEC-1980", 9));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "12/17/1980", 9));
+    // Dates and timestamps are forbidden
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2023-01-19", 9));
+    expectError(
+        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2023-01-19T14:23:55.878137", 9));
 
     // Test forbidden values
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", LocalDate.now(), 3));
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", LocalDateTime.now(), 3));
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", OffsetDateTime.now(), 3));
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", ZonedDateTime.now(), 3));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", Instant.now(), 3));
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", new Date(), 3));
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", 1.5f, 3));
     expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", 1.5, 3));
@@ -181,132 +215,64 @@ public void testValidateAndParseTime() {
   }
 
   @Test
-  public void testValidateAndParseTimestampNtzSb16() {
-    assertEquals(
-        new TimestampWrapper(
-            1609462800,
-            123000000,
-            new BigInteger("1609462800123000000"),
-            SFTimestamp.fromNanoseconds(
-                BigDecimal.valueOf(1609462800 * 1_000_000_000L + 123000000))),
-        DataValidationUtil.validateAndParseTimestamp("COL", "2021-01-01 01:00:00.123", 9, true));
-
-    // Time formats are not supported
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestamp("COL", "20:57:01.123456789+07:00", 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestamp("COL", "20:57:01.123456789", 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "20:57:01", 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "20:57", 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestamp("COL", "07:57:01.123456789 AM", 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "04:01:07 AM", 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "04:01 AM", 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "04:01 PM", 3, false));
-
-    // Test forbidden values
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", LocalTime.now(), 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", OffsetTime.now(), 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", new Date(), 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 1.5f, 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 1.5, 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "1.5", 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "1.0", 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", new Object(), 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", false, 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "", 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "foo", 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestamp("COL", java.sql.Time.valueOf("20:57:00"), 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestamp("COL", java.sql.Date.valueOf("2010-11-03"), 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () ->
-            validateAndParseTimestamp(
-                "COL", java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", BigInteger.ZERO, 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", BigDecimal.ZERO, 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 'c', 3, false));
-  }
-
-  @Test
-  public void testValidateAndPareTimestampTz() {
-    TimestampWrapper result =
+  public void testValidateAndParseTimestamp() {
+    TimestampWrapper wrapper =
         DataValidationUtil.validateAndParseTimestamp(
-            "COL", "2021-01-01 01:00:00.123 +0100", 4, false);
-    assertEquals(1609459200, result.getEpoch());
-    assertEquals(123000000, result.getFraction());
-    assertEquals(Optional.of(3600000), result.getTimezoneOffset());
-    assertEquals(Optional.of(1500), result.getTimeZoneIndex());
+            "COL", "2021-01-01T01:00:00.123+01:00", 4, UTC, false);
+    assertEquals(1609459200, wrapper.getEpoch());
+    assertEquals(123000000, wrapper.getFraction());
+    assertEquals(3600, wrapper.getTimezoneOffsetSeconds());
+    assertEquals(1500, wrapper.getTimeZoneIndex());
+
+    wrapper = validateAndParseTimestamp("COL", "2021-01-01T01:00:00.123", 9, UTC, true);
+    Assert.assertEquals(1609462800, wrapper.getEpoch());
+    Assert.assertEquals(123000000, wrapper.getFraction());
+    Assert.assertEquals(new BigInteger("1609462800123000000"), wrapper.toBinary(false));
 
-    // Time formats are not supported
+    // Time input is not supported
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestamp("COL", "20:57:01.123456789+07:00", 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "20:57:01", 3, UTC, false));
+
+    // Test forbidden values
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestamp("COL", "20:57:01.123456789", 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "20:57:01", 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "20:57", 3, false));
+        () -> validateAndParseTimestamp("COL", LocalTime.now(), 3, UTC, false));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestamp("COL", "07:57:01.123456789 AM", 3, false));
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "04:01:07 AM", 3, false));
+        () -> validateAndParseTimestamp("COL", OffsetTime.now(), 3, UTC, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "04:01 AM", 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", new Date(), 3, UTC, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 1.5f, 3, UTC, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 1.5, 3, UTC, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "04:01 PM", 3, false));
-
-    // Test forbidden values
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "1.5", 3, UTC, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", LocalTime.now(), 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "1.0", 3, UTC, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", OffsetTime.now(), 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", new Date(), 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 1.5f, 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 1.5, 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "1.5", 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "1.0", 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", false, 3, UTC, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "", 3, UTC, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", new Object(), 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", false, 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "", 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "foo", 3, false));
+        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "foo", 3, UTC, false));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestamp("COL", java.sql.Time.valueOf("20:57:00"), 3, false));
+        () -> validateAndParseTimestamp("COL", java.sql.Time.valueOf("20:57:00"), 3, UTC, false));
     expectError(
         ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestamp("COL", java.sql.Date.valueOf("2010-11-03"), 3, false));
+        () -> validateAndParseTimestamp("COL", java.sql.Date.valueOf("2010-11-03"), 3, UTC, false));
     expectError(
         ErrorCode.INVALID_ROW,
         () ->
             validateAndParseTimestamp(
-                "COL", java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3, false));
+                "COL", java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3, UTC, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", BigInteger.ZERO, 3, false));
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestamp("COL", BigInteger.ZERO, 3, UTC, false));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", BigDecimal.ZERO, 3, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 'c', 3, false));
+        ErrorCode.INVALID_ROW,
+        () -> validateAndParseTimestamp("COL", BigDecimal.ZERO, 3, UTC, false));
+    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 'c', 3, UTC, false));
   }
 
   @Test
@@ -780,41 +746,6 @@ public void testValidVariantType() {
                     Arrays.asList(13, Arrays.asList(Arrays.asList(new Object(), 'c')))))));
   }
 
-  @Test
-  public void testValidateAndParseDate() {
-    assertEquals(-923, validateAndParseDate("COL", "1967-06-23"));
-    assertEquals(-923, validateAndParseDate("COL", "1967-06-23 01:01:01"));
-    assertEquals(18464, validateAndParseDate("COL", "2020-07-21"));
-    assertEquals(18464, validateAndParseDate("COL", "2020-07-21 23:31:00"));
-
-    // Time formats are not supported
-    expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "20:57:01.123456789+07:00"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "20:57:01.123456789"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "20:57:01"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "20:57"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "07:57:01.123456789 AM"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "04:01:07 AM"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "04:01 AM"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "04:01 PM"));
-
-    // Test forbidden values
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", new Object()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", LocalTime.now()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", OffsetTime.now()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", new java.util.Date()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", ""));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "foo"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "1.0"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", 'c'));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", 1));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", 1L));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", 1.25));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", BigInteger.valueOf(1)));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", BigDecimal.valueOf(1.25)));
-  }
-
   @Test
   public void testValidateAndParseBinary() {
     byte[] maxAllowedArray = new byte[BYTES_8_MB];
@@ -951,7 +882,7 @@ public void testExceptionMessages() {
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
             + " Snowflake column COL of type TIME: Not a valid time, see"
-            + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#time-formats"
+            + " https://docs.snowflake.com/en/LIMITEDACCESS/snowpipe-streaming.html"
             + " for the list of supported formats",
         () -> validateAndParseTime("COL", "abc", 10));
 
@@ -965,9 +896,9 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column COL of type DATE: Not a valid date, see"
-            + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#date-formats"
-            + " for the list of supported formats",
+            + " Snowflake column COL of type DATE: Not a valid value, see"
+            + " https://docs.snowflake.com/en/LIMITEDACCESS/snowpipe-streaming.html for the list of"
+            + " supported formats",
         () -> validateAndParseDate("COL", "abc"));
 
     // TIMESTAMP_NTZ
@@ -976,14 +907,14 @@ public void testExceptionMessages() {
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
             + " be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
             + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
-        () -> validateAndParseTimestamp("COL", new Object(), 3, true));
+        () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, true));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column COL of type TIMESTAMP: Not a valid timestamp, see"
-            + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
-            + " for the list of supported formats",
-        () -> validateAndParseTimestamp("COL", "abc", 3, true));
+            + " Snowflake column COL of type TIMESTAMP: Not a valid value, see"
+            + " https://docs.snowflake.com/en/LIMITEDACCESS/snowpipe-streaming.html for the list of"
+            + " supported formats",
+        () -> validateAndParseTimestamp("COL", "abc", 3, UTC, true));
 
     // TIMESTAMP_LTZ
     expectErrorCodeAndMessage(
@@ -991,14 +922,14 @@ public void testExceptionMessages() {
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
             + " be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
             + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
-        () -> validateAndParseTimestamp("COL", new Object(), 3, false));
+        () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, false));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column COL of type TIMESTAMP: Not a valid timestamp, see"
-            + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
-            + " for the list of supported formats",
-        () -> validateAndParseTimestamp("COL", "abc", 3, false));
+            + " Snowflake column COL of type TIMESTAMP: Not a valid value, see"
+            + " https://docs.snowflake.com/en/LIMITEDACCESS/snowpipe-streaming.html for the list of"
+            + " supported formats",
+        () -> validateAndParseTimestamp("COL", "abc", 3, UTC, false));
 
     // TIMESTAMP_TZ
     expectErrorCodeAndMessage(
@@ -1006,14 +937,14 @@ public void testExceptionMessages() {
         "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
             + " be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
             + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
-        () -> validateAndParseTimestamp("COL", new Object(), 3, false));
+        () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, false));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column COL of type TIMESTAMP: Not a valid timestamp, see"
-            + " https://docs.snowflake.com/en/user-guide/date-time-input-output.html#timestamp-formats"
-            + " for the list of supported formats",
-        () -> validateAndParseTimestamp("COL", "abc", 3, false));
+            + " Snowflake column COL of type TIMESTAMP: Not a valid value, see"
+            + " https://docs.snowflake.com/en/LIMITEDACCESS/snowpipe-streaming.html for the list of"
+            + " supported formats",
+        () -> validateAndParseTimestamp("COL", "abc", 3, UTC, false));
 
     // NUMBER
     expectErrorCodeAndMessage(
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index b4554f481..90344b8a6 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -19,6 +19,8 @@
 import java.security.InvalidAlgorithmParameterException;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Base64;
@@ -122,7 +124,8 @@ abstract SnowflakeStreamingIngestChannelInternal<T> createChannel(
         Long rowSequencer,
         String encryptionKey,
         Long encryptionKeyId,
-        OpenChannelRequest.OnErrorOption onErrorOption);
+        OpenChannelRequest.OnErrorOption onErrorOption,
+        ZoneId defaultTimezone);
 
     ChannelBuilder channelBuilder(String name) {
       return new ChannelBuilder(name);
@@ -197,7 +200,8 @@ SnowflakeStreamingIngestChannelInternal<T> buildAndAdd() {
                 rowSequencer,
                 encryptionKey,
                 encryptionKeyId,
-                onErrorOption);
+                onErrorOption,
+                ZoneOffset.UTC);
         channels.put(name, channel);
         channelCache.addChannel(channel);
         return channel;
@@ -246,7 +250,8 @@ SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> createChannel(
         Long rowSequencer,
         String encryptionKey,
         Long encryptionKeyId,
-        OpenChannelRequest.OnErrorOption onErrorOption) {
+        OpenChannelRequest.OnErrorOption onErrorOption,
+        ZoneId defaultTimezone) {
       return new SnowflakeStreamingIngestChannelInternal<>(
           name,
           dbName,
@@ -259,6 +264,7 @@ SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> createChannel(
           encryptionKey,
           encryptionKeyId,
           onErrorOption,
+          defaultTimezone,
           Constants.BdecVersion.ONE,
           allocator);
     }
@@ -295,7 +301,8 @@ SnowflakeStreamingIngestChannelInternal<List<List<Object>>> createChannel(
         Long rowSequencer,
         String encryptionKey,
         Long encryptionKeyId,
-        OpenChannelRequest.OnErrorOption onErrorOption) {
+        OpenChannelRequest.OnErrorOption onErrorOption,
+        ZoneId defaultTimezone) {
       return new SnowflakeStreamingIngestChannelInternal<>(
           name,
           dbName,
@@ -308,6 +315,7 @@ SnowflakeStreamingIngestChannelInternal<List<List<Object>>> createChannel(
           encryptionKey,
           encryptionKeyId,
           onErrorOption,
+          defaultTimezone,
           Constants.BdecVersion.THREE,
           null);
     }
@@ -638,7 +646,8 @@ public void testInvalidateChannels() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            ZoneOffset.UTC);
 
     SnowflakeStreamingIngestChannelInternal<StubChunkData> channel2 =
         new SnowflakeStreamingIngestChannelInternal<>(
@@ -652,7 +661,8 @@ public void testInvalidateChannels() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            ZoneOffset.UTC);
 
     channelCache.addChannel(channel1);
     channelCache.addChannel(channel2);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
index cc0361147..15be2cdb2 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
@@ -1,5 +1,7 @@
 package net.snowflake.ingest.streaming.internal;
 
+import static java.time.ZoneOffset.UTC;
+
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
@@ -26,7 +28,7 @@ public void parseValueFixedSB1ToInt32() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            12, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+            12, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats, UTC);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -52,7 +54,7 @@ public void parseValueFixedSB2ToInt32() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            1234, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+            1234, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats, UTC);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -78,7 +80,7 @@ public void parseValueFixedSB4ToInt32() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            123456789, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+            123456789, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats, UTC);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -104,7 +106,11 @@ public void parseValueFixedSB8ToInt64() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            123456789987654321L, testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats);
+            123456789987654321L,
+            testCol,
+            PrimitiveType.PrimitiveTypeName.INT64,
+            rowBufferStats,
+            UTC);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -133,7 +139,8 @@ public void parseValueFixedSB16ToByteArray() {
             new BigDecimal("91234567899876543219876543211234567891"),
             testCol,
             PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
-            rowBufferStats);
+            rowBufferStats,
+            UTC);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -164,7 +171,8 @@ public void parseValueFixedDecimalToInt32() {
             new BigDecimal("12345.54321"),
             testCol,
             PrimitiveType.PrimitiveTypeName.DOUBLE,
-            rowBufferStats);
+            rowBufferStats,
+            UTC);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -188,7 +196,7 @@ public void parseValueDouble() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            12345.54321d, testCol, PrimitiveType.PrimitiveTypeName.DOUBLE, rowBufferStats);
+            12345.54321d, testCol, PrimitiveType.PrimitiveTypeName.DOUBLE, rowBufferStats, UTC);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -212,7 +220,7 @@ public void parseValueBoolean() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            true, testCol, PrimitiveType.PrimitiveTypeName.BOOLEAN, rowBufferStats);
+            true, testCol, PrimitiveType.PrimitiveTypeName.BOOLEAN, rowBufferStats, UTC);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -236,7 +244,11 @@ public void parseValueBinary() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            "1234abcd".getBytes(), testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
+            "1234abcd".getBytes(),
+            testCol,
+            PrimitiveType.PrimitiveTypeName.BINARY,
+            rowBufferStats,
+            UTC);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -272,7 +284,7 @@ private void testJsonWithLogicalType(String logicalType) {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            var, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
+            var, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats, UTC);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -310,7 +322,7 @@ private void testNullJsonWithLogicalType(String var) {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            var, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
+            var, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats, UTC);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -340,7 +352,7 @@ public void parseValueArrayToBinary() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            input, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats);
+            input, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats, UTC);
 
     String resultArray = "[{\"a\":\"1\",\"b\":\"2\",\"c\":\"3\"}]";
 
@@ -374,7 +386,8 @@ public void parseValueTimestampNtzSB4Error() {
                     "2013-04-28 20:57:00",
                     testCol,
                     PrimitiveType.PrimitiveTypeName.INT32,
-                    rowBufferStats));
+                    rowBufferStats,
+                    UTC));
     Assert.assertEquals(
         "Unknown data type for logical: TIMESTAMP_NTZ, physical: SB4.", exception.getMessage());
   }
@@ -393,10 +406,11 @@ public void parseValueTimestampNtzSB8ToINT64() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            "2013-04-28 20:57:01.000",
+            "2013-04-28T20:57:01.000",
             testCol,
             PrimitiveType.PrimitiveTypeName.INT64,
-            rowBufferStats);
+            rowBufferStats,
+            UTC);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -421,10 +435,11 @@ public void parseValueTimestampNtzSB16ToByteArray() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            "2022-09-18 22:05:07.123456789",
+            "2022-09-18T22:05:07.123456789",
             testCol,
             PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
-            rowBufferStats);
+            rowBufferStats,
+            UTC);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -450,7 +465,7 @@ public void parseValueDateToInt32() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            "2021-01-01", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+            "2021-01-01", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats, UTC);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -475,7 +490,7 @@ public void parseValueTimeSB4ToInt32() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            "01:00:00", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats);
+            "01:00:00", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats, UTC);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -500,7 +515,7 @@ public void parseValueTimeSB8ToInt64() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            "01:00:00.123", testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats);
+            "01:00:00.123", testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats, UTC);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -531,7 +546,8 @@ public void parseValueTimeSB16Error() {
                     "11:00:00.12345678",
                     testCol,
                     PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
-                    rowBufferStats));
+                    rowBufferStats,
+                    UTC));
     Assert.assertEquals(
         "Unknown data type for logical: TIME, physical: SB16.", exception.getMessage());
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index b867acf95..00e92c76b 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -1,5 +1,7 @@
 package net.snowflake.ingest.streaming.internal;
 
+import static java.time.ZoneOffset.UTC;
+
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
@@ -119,6 +121,7 @@ private AbstractRowBuffer<?> createTestBuffer(OpenChannelRequest.OnErrorOption o
     ChannelRuntimeState initialState = new ChannelRuntimeState("0", 0L, true);
     return AbstractRowBuffer.createRowBuffer(
         onErrorOption,
+        UTC,
         new RootAllocator(),
         bdecVersion,
         "test.buffer",
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index bb328aa08..28aca8d77 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -1,5 +1,6 @@
 package net.snowflake.ingest.streaming.internal;
 
+import static java.time.ZoneOffset.UTC;
 import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
 import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
 import static net.snowflake.ingest.utils.Constants.OPEN_CHANNEL_ENDPOINT;
@@ -10,6 +11,7 @@
 
 import java.security.KeyPair;
 import java.security.PrivateKey;
+import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -55,7 +57,9 @@ public void testChannelFactoryNullFields() {
         new SnowflakeStreamingIngestClientInternal<>("client");
 
     Object[] fields =
-        new Object[] {name, dbName, schemaName, tableName, channelSequencer, rowSequencer, client};
+        new Object[] {
+          name, dbName, schemaName, tableName, channelSequencer, rowSequencer, client, UTC
+        };
 
     for (int i = 0; i < fields.length; i++) {
       Object tmp = fields[i];
@@ -68,6 +72,7 @@ public void testChannelFactoryNullFields() {
             .setRowSequencer((Long) fields[4])
             .setChannelSequencer((Long) fields[5])
             .setOwningClient((SnowflakeStreamingIngestClientInternal<StubChunkData>) fields[6])
+            .setDefaultTimezone((ZoneId) fields[7])
             .build();
         Assert.fail("Channel factory should fail with null fields");
       } catch (SFException e) {
@@ -105,6 +110,7 @@ public void testChannelFactorySuccess() {
             .setEncryptionKey(encryptionKey)
             .setEncryptionKeyId(1234L)
             .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+            .setDefaultTimezone(UTC)
             .build();
 
     Assert.assertEquals(name, channel.getName());
@@ -138,7 +144,8 @@ public void testChannelValid() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
 
     Assert.assertTrue(channel.isValid());
     channel.invalidate("from testChannelValid");
@@ -187,7 +194,8 @@ public void testChannelClose() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
 
     Assert.assertFalse(channel.isClosed());
     channel.markClosed();
@@ -484,7 +492,8 @@ public void testInsertRow() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
 
     ColumnMetadata col = new ColumnMetadata();
     col.setName("COL");
@@ -543,7 +552,8 @@ public void testInsertRowThrottling() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
 
     Runtime mockedRunTime = Mockito.mock(Runtime.class);
     Mockito.when(mockedRunTime.maxMemory()).thenReturn(maxMemory);
@@ -590,7 +600,8 @@ public void testFlush() throws Exception {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
     ChannelsStatusResponse response = new ChannelsStatusResponse();
     response.setStatusCode(0L);
     response.setMessage("Success");
@@ -625,7 +636,8 @@ public void testClose() throws Exception {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
     ChannelsStatusResponse response = new ChannelsStatusResponse();
     response.setStatusCode(0L);
     response.setMessage("Success");
@@ -658,7 +670,8 @@ public void testGetLatestCommittedOffsetToken() {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
 
     ChannelsStatusResponse response = new ChannelsStatusResponse();
     response.setStatusCode(0L);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index 7d5e52818..93b15398b 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -1,5 +1,6 @@
 package net.snowflake.ingest.streaming.internal;
 
+import static java.time.ZoneOffset.UTC;
 import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
 import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
@@ -20,6 +21,7 @@
 import java.security.KeyPair;
 import java.security.PrivateKey;
 import java.security.Security;
+import java.time.ZoneOffset;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -89,6 +91,7 @@ public void setup() {
             "key",
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
+            ZoneOffset.UTC,
             Constants.BdecVersion.ONE,
             new RootAllocator());
     channel2 =
@@ -104,6 +107,7 @@ public void setup() {
             "key",
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
+            ZoneOffset.UTC,
             Constants.BdecVersion.ONE,
             new RootAllocator());
     channel3 =
@@ -119,6 +123,7 @@ public void setup() {
             "key",
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
+            ZoneOffset.UTC,
             Constants.BdecVersion.ONE,
             new RootAllocator());
     channel4 =
@@ -134,6 +139,7 @@ public void setup() {
             "key",
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
+            ZoneOffset.UTC,
             Constants.BdecVersion.ONE,
             new RootAllocator());
   }
@@ -351,6 +357,7 @@ public void testGetChannelsStatusWithRequest() throws Exception {
             "key",
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
+            ZoneOffset.UTC,
             Constants.BdecVersion.ONE,
             new RootAllocator());
 
@@ -410,6 +417,7 @@ public void testGetChannelsStatusWithRequestError() throws Exception {
             "key",
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
+            ZoneOffset.UTC,
             Constants.BdecVersion.ONE,
             new RootAllocator());
 
@@ -449,6 +457,7 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
             "key",
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
+            ZoneOffset.UTC,
             Constants.BdecVersion.ONE,
             new RootAllocator());
 
@@ -1050,7 +1059,8 @@ public void testRegisterBlobResponseWithInvalidChannel() throws Exception {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
     SnowflakeStreamingIngestChannelInternal<StubChunkData> channel2 =
         new SnowflakeStreamingIngestChannelInternal<>(
             channel2Name,
@@ -1063,7 +1073,8 @@ public void testRegisterBlobResponseWithInvalidChannel() throws Exception {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
     client.getChannelCache().addChannel(channel1);
     client.getChannelCache().addChannel(channel2);
 
@@ -1189,7 +1200,8 @@ public void testVerifyChannelsAreFullyCommittedSuccess() throws Exception {
             client,
             "key",
             1234L,
-            OpenChannelRequest.OnErrorOption.CONTINUE);
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
     client.getChannelCache().addChannel(channel);
 
     ChannelsStatusResponse response = new ChannelsStatusResponse();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index aa48c07f7..8c936bac4 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -361,26 +361,26 @@ public void testTimeColumnIngest() throws Exception {
     SnowflakeStreamingIngestChannel channel1 = client.openChannel(request1);
 
     Map<String, Object> row = new HashMap<>();
-    row.put("ttzsmall", "2021-01-01 01:00:00.123 -0300");
-    row.put("ttzbig", "2021-01-01 09:00:00.12345678 -0300");
+    row.put("ttzsmall", "2021-01-01T01:00:00.123-03:00");
+    row.put("ttzbig", "2021-01-01T09:00:00.12345678-03:00");
     row.put("tsmall", "01:00:00.123");
     row.put("tbig", "09:00:00.12345678");
     row.put("tntzsmall", "1609462800123");
     row.put("tntzbig", "1609462800123450000");
     verifyInsertValidationResponse(channel1.insertRow(row, null));
-    row.put("ttzsmall", "2021-01-01 10:00:00.123 +0700");
-    row.put("ttzbig", "2021-01-01 19:00:00.12345678 -0300");
+    row.put("ttzsmall", "2021-01-01T10:00:00.123+07:00");
+    row.put("ttzbig", "2021-01-01T19:00:00.12345678-03:00");
     row.put("tsmall", "02:00:00.123");
     row.put("tbig", "10:00:00.12345678");
     row.put("tntzsmall", "1709462800123");
     row.put("tntzbig", "170946280212345000");
     verifyInsertValidationResponse(channel1.insertRow(row, null));
-    row.put("ttzsmall", "2021-01-01 05:00:00 +0100");
-    row.put("ttzbig", "2021-01-01 23:00:00.12345678 -0300");
+    row.put("ttzsmall", "2021-01-01T05:00:00+01:00");
+    row.put("ttzbig", "2021-01-01T23:00:00.12345678-03:00");
     row.put("tsmall", "03:00:00.123");
     row.put("tbig", "11:00:00.12345678");
     row.put("tntzsmall", "1809462800123");
-    row.put("tntzbig", "2031-01-01 09:00:00.123456780");
+    row.put("tntzbig", "2031-01-01T09:00:00.123456780");
     verifyInsertValidationResponse(channel1.insertRow(row, "1"));
 
     // Close the channel after insertion
@@ -472,7 +472,7 @@ public void testMultiColumnIngest() throws Exception {
     row.put("tinyfloat", 1.1);
     row.put("var", "{\"e\":2.7}");
     row.put("t", String.valueOf(timestamp));
-    row.put("d", "1969-12-31 00:00:00");
+    row.put("d", "1969-12-31T00:00:00");
     verifyInsertValidationResponse(channel1.insertRow(row, "1"));
 
     // Close the channel after insertion
@@ -1230,9 +1230,9 @@ private Map<String, Object> getPosRow() {
         "{ \"a\": 1, \"b\": \"qwerty\", \"c\": null, \"d\": { \"e\": 2, \"f\": \"asdf\", \"g\":"
             + " null } }");
     posRow.put("arr", Arrays.asList("{ \"a\": 1}", "{ \"b\": 2 }", "{ \"c\": 3 }"));
-    posRow.put("epochdays", "2022-09-18 20:05:07"); // DATE, 18.09.2022
-    posRow.put("epochsec", "2022-09-18 20:05:07"); // TIMESTAMP_NTZ(0)
-    posRow.put("epochnano", "2022-09-18 20:05:07.999999999"); // TIMESTAMP_NTZ(9)
+    posRow.put("epochdays", "2022-09-18T20:05:07"); // DATE, 18.09.2022
+    posRow.put("epochsec", "2022-09-18T20:05:07"); // TIMESTAMP_NTZ(0)
+    posRow.put("epochnano", "2022-09-18T20:05:07.999999999"); // TIMESTAMP_NTZ(9)
     posRow.put("timesec", "01:00:01.999999999"); // TIME(0)
     posRow.put("timenano", "01:00:01.999999999"); // TIME(9)
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index 729f3d354..f3b575527 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -9,9 +9,11 @@
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.time.ZoneId;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Properties;
 import java.util.UUID;
 import java.util.function.Predicate;
@@ -53,6 +55,12 @@ public static Collection<Object[]> bdecVersion() {
   protected Connection conn;
   private String databaseName;
 
+  /**
+   * Default timezone for newly created channels. If empty, the test will not explicitly set default
+   * timezone in the channel builder.
+   */
+  private Optional<ZoneId> defaultTimezone = Optional.empty();
+
   private String schemaName = "PUBLIC";
   private SnowflakeStreamingIngestClient client;
   private static final ObjectMapper objectMapper = new ObjectMapper();
@@ -72,9 +80,6 @@ public void before() throws Exception {
     conn.createStatement().execute(String.format("use database %s;", databaseName));
     conn.createStatement().execute(String.format("use schema %s;", schemaName));
 
-    // Set to a random time zone not to interfere with any of the tests
-    conn.createStatement().execute("alter session set timezone = 'America/New_York';");
-
     conn.createStatement().execute(String.format("use warehouse %s;", TestUtils.getWarehouse()));
 
     Properties props = TestUtils.getProperties(bdecVersion);
@@ -86,6 +91,7 @@ public void before() throws Exception {
 
   @After
   public void after() throws Exception {
+    this.defaultTimezone = Optional.empty();
     conn.createStatement().executeQuery(String.format("drop database %s", databaseName));
     if (client != null) {
       client.close();
@@ -95,6 +101,10 @@ public void after() throws Exception {
     }
   }
 
+  void setChannelDefaultTimezone(ZoneId defaultTimezone) {
+    this.defaultTimezone = Optional.of(defaultTimezone);
+  }
+
   protected String createTable(String dataType) throws SQLException {
     String tableName = getRandomIdentifier();
     conn.createStatement()
@@ -115,13 +125,14 @@ protected SnowflakeStreamingIngestChannel openChannel(String tableName) {
 
   protected SnowflakeStreamingIngestChannel openChannel(
       String tableName, OpenChannelRequest.OnErrorOption onErrorOption) {
-    OpenChannelRequest openChannelRequest =
+    OpenChannelRequest.OpenChannelRequestBuilder requestBuilder =
         OpenChannelRequest.builder("CHANNEL")
             .setDBName(databaseName)
             .setSchemaName(SCHEMA_NAME)
             .setTableName(tableName)
-            .setOnErrorOption(onErrorOption)
-            .build();
+            .setOnErrorOption(onErrorOption);
+    defaultTimezone.ifPresent(requestBuilder::setDefaultTimezone);
+    OpenChannelRequest openChannelRequest = requestBuilder.build();
     return client.openChannel(openChannelRequest);
   }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
index 4abbf29db..8a8d9bafc 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
@@ -6,649 +6,521 @@
 import java.time.LocalTime;
 import java.time.OffsetDateTime;
 import java.time.OffsetTime;
+import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import net.snowflake.ingest.utils.Constants;
-import org.junit.Ignore;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
-/**
- * Supported date, time and timestamp formats:
- * https://docs.snowflake.com/en/user-guide/date-time-input-output.html#date-formats
- */
 public class DateTimeIT extends AbstractDataTypeTest {
 
+  private static final ZoneId TZ_LOS_ANGELES = ZoneId.of("America/Los_Angeles");
+  private static final ZoneId TZ_BERLIN = ZoneId.of("Europe/Berlin");
+  private static final ZoneId TZ_TOKYO = ZoneId.of("Asia/Tokyo");
+
   public DateTimeIT(String name, Constants.BdecVersion bdecVersion) {
     super(name, bdecVersion);
   }
 
+  @Before
+  public void setup() throws Exception {
+    // Set to a random time zone not to interfere with any of the tests
+    conn.createStatement().execute("alter session set timezone = 'America/New_York';");
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    conn.createStatement().execute("alter session unset timezone;");
+  }
+
+  @Test
+  public void testDefaultTimezone() throws Exception {
+    testIngestion(
+        "TIMESTAMP_TZ", "2000-01-01", "2000-01-01 00:00:00.000000000 -0800", new StringProvider());
+    testIngestion(
+        "TIMESTAMP_LTZ", "2000-01-01", "2000-01-01 03:00:00.000000000 -0500", new StringProvider());
+  }
+
   @Test
   public void testTimestampWithTimeZone() throws Exception {
-    useLosAngelesTimeZone();
+    setJdbcSessionTimezone(TZ_LOS_ANGELES);
+    setChannelDefaultTimezone(TZ_LOS_ANGELES);
 
     // Test timestamp formats
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "2/18/2008 02:36:48",
-        "2008-02-18 02:36:48.000000000 -0800",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ",
-        "2013-04-28 20",
-        "2013-04-28 20:00:00.000000000 -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ",
-        "2013-04-28 20:57",
+        "2013-04-28T20:57",
         "2013-04-28 20:57:00.000000000 -0700",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "2013-04-28 20:57:01",
-        "2013-04-28 20:57:01.000000000 -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ",
-        "2013-04-28 20:57:01 +07:00",
-        "2013-04-28 20:57:01.000000000 +0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ",
-        "2013-04-28 20:57:01 +0700",
-        "2013-04-28 20:57:01.000000000 +0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ",
-        "2013-04-28 20:57:01-07",
+        "2013-04-28T20:57:01",
         "2013-04-28 20:57:01.000000000 -0700",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "2013-04-28 20:57:01-07:00",
+        "2013-04-28T20:57:01-07:00",
         "2013-04-28 20:57:01.000000000 -0700",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "2013-04-28 20:57:01.123456",
+        "2013-04-28T20:57:01.123456",
         "2013-04-28 20:57:01.123456000 -0700",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "2013-04-28 20:57:01.123456789 +07:00",
+        "2013-04-28T20:57:01.123456789+07:00",
         "2013-04-28 20:57:01.123456789 +0700",
         new StringProvider(),
         new StringProvider());
-    testJdbcTypeCompatibility(
+    // Here we test ingestion only because JDBC cannot ingest ISO_ZONED_DATE_TIME
+    testIngestion(
         "TIMESTAMP_TZ",
-        "2013-04-28 20:57:01.123456789 +0700",
-        "2013-04-28 20:57:01.123456789 +0700",
-        new StringProvider(),
+        "2013-04-28T20:57:01.123456789+09:00[Asia/Tokyo]",
+        "2013-04-28 20:57:01.123456789 +0900",
         new StringProvider());
+
+    // Test date formats
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "2013-04-28 20:57:01.123456789+07",
-        "2013-04-28 20:57:01.123456789 +0700",
+        "2013-04-28",
+        "2013-04-28 00:00:00.000000000 -0700",
         new StringProvider(),
         new StringProvider());
+
+    // Test leap years
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "2013-04-28 20:57:01.123456789+07:00",
-        "2013-04-28 20:57:01.123456789 +0700",
+        "2024-02-29T23:59:59.999999999Z",
+        "2024-02-29 23:59:59.999999999 Z",
         new StringProvider(),
         new StringProvider());
+    expectArrowNotSupported("TIMESTAMP_TZ", "2023-02-29T23:59:59.999999999");
+
+    // Numeric strings
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "2013-04-28 20:57+07:00",
-        "2013-04-28 20:57:00.000000000 +0700",
+        "0",
+        "1970-01-01 00:00:00.000000000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "2013-04-28T20",
-        "2013-04-28 20:00:00.000000000 -0700",
+        "1674478926",
+        "2023-01-23 13:02:06.000000000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "2013-04-28T20:57",
-        "2013-04-28 20:57:00.000000000 -0700",
+        "1674478926123",
+        "2023-01-23 13:02:06.123000000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "2013-04-28T20:57:01",
-        "2013-04-28 20:57:01.000000000 -0700",
+        "1674478926123456",
+        "2023-01-23 13:02:06.123456000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "2013-04-28T20:57:01-07:00",
-        "2013-04-28 20:57:01.000000000 -0700",
+        "1674478926123456789",
+        "2023-01-23 13:02:06.123456789 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "2013-04-28T20:57:01.123456",
-        "2013-04-28 20:57:01.123456000 -0700",
+        "-1674478926",
+        "1916-12-09 10:57:54.000000000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "2013-04-28T20:57:01.123456789+07:00",
-        "2013-04-28 20:57:01.123456789 +0700",
+        "-1674478926123",
+        "1916-12-09 10:57:53.877000000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "2013-04-28T20:57+07:00",
-        "2013-04-28 20:57:00.000000000 +0700",
+        "-1674478926123456",
+        "1916-12-09 10:57:53.876544000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "Mon Jul 08 18:09:51 +0000 2013",
-        "2013-07-08 18:09:51.000000000 Z",
+        "-1674478926123456789",
+        "1916-12-09 10:57:53.876543211 Z",
         new StringProvider(),
         new StringProvider());
-    testJdbcTypeCompatibility(
+
+    // java.time.LocalDate
+    testIngestion(
         "TIMESTAMP_TZ",
-        "Thu, 21 Dec 2000 04:01:07 PM",
-        "2000-12-21 16:01:07.000000000 -0800",
-        new StringProvider(),
+        LocalDate.parse("2007-12-03"),
+        "2007-12-03 00:00:00.000000000 -0800",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    // java.time.LocalDateTime
+    testIngestion(
         "TIMESTAMP_TZ",
-        "Thu, 21 Dec 2000 04:01:07 PM +0200",
-        "2000-12-21 16:01:07.000000000 +0200",
-        new StringProvider(),
+        LocalDateTime.parse("2007-12-03T10:15:30"),
+        "2007-12-03 10:15:30.000000000 -0800",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    // java.time.OffsetDateTime
+    testIngestion(
         "TIMESTAMP_TZ",
-        "Thu, 21 Dec 2000 04:01:07.123456789 PM",
-        "2000-12-21 16:01:07.123456789 -0800",
-        new StringProvider(),
+        OffsetDateTime.parse("2007-12-03T10:15:30+01:00"),
+        "2007-12-03 10:15:30.000000000 +0100",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    // java.time.ZonedDateTime
+    testIngestion(
         "TIMESTAMP_TZ",
-        "Thu, 21 Dec 2000 04:01:07.123456789 PM +0200",
-        "2000-12-21 16:01:07.123456789 +0200",
-        new StringProvider(),
+        ZonedDateTime.parse("2007-12-03T10:15:30.000456789+01:00[Europe/Paris]"),
+        "2007-12-03 10:15:30.000456789 +0100",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    // java.time.Instant
+    testIngestion(
         "TIMESTAMP_TZ",
-        "Thu, 21 Dec 2000 16:01:07",
-        "2000-12-21 16:01:07.000000000 -0800",
-        new StringProvider(),
+        OffsetDateTime.parse("2007-12-03T10:15:30.123456789+00:00").toInstant(),
+        "2007-12-03 10:15:30.123456789 Z",
         new StringProvider());
-    testJdbcTypeCompatibility(
+
+    // Verify that default timezone has impact on input without timezone information
+    setChannelDefaultTimezone(TZ_BERLIN);
+    testIngestion(
         "TIMESTAMP_TZ",
-        "Thu, 21 Dec 2000 16:01:07 +0200",
-        "2000-12-21 16:01:07.000000000 +0200",
-        new StringProvider(),
+        "2013-04-28T20:57:01.123456789",
+        "2013-04-28 20:57:01.123456789 +0200",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    testIngestion(
         "TIMESTAMP_TZ",
-        "Thu, 21 Dec 2000 16:01:07.123456789",
-        "2000-12-21 16:01:07.123456789 -0800",
-        new StringProvider(),
+        LocalDate.parse("2007-12-03"),
+        "2007-12-03 00:00:00.000000000 +0100",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    testIngestion(
         "TIMESTAMP_TZ",
-        "Thu, 21 Dec 2000 16:01:07.123456789 +0200",
-        "2000-12-21 16:01:07.123456789 +0200",
-        new StringProvider(),
+        LocalDateTime.parse("2007-12-03T10:15:30"),
+        "2007-12-03 10:15:30.000000000 +0100",
         new StringProvider());
 
-    // Test date formats
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ",
-        "2013-04-28",
-        "2013-04-28 00:00:00.000000000 -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
+    // No impact on integer-stored and instant as they are always UTC
+    testIngestion(
         "TIMESTAMP_TZ",
-        "17-DEC-1980",
-        "1980-12-17 00:00:00.000000000 -0800",
-        new StringProvider(),
+        "1674478926123456789",
+        "2023-01-23 13:02:06.123456789 Z",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    testIngestion(
         "TIMESTAMP_TZ",
-        "12/17/1980",
-        "1980-12-17 00:00:00.000000000 -0800",
-        new StringProvider(),
+        OffsetDateTime.parse("2007-12-03T10:15:30.123456789+00:00").toInstant(),
+        "2007-12-03 10:15:30.123456789 Z",
         new StringProvider());
 
-    // Test leap years
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ",
-        "2024-02-29T23:59:59.999999999Z",
-        "2024-02-29 23:59:59.999999999 Z",
-        new StringProvider(),
+    // Limited scale
+    testIngestion(
+        "TIMESTAMP_TZ(0)",
+        "2022-01-31T13:00:00+09:00",
+        "2022-01-31 13:00:00. +0900",
         new StringProvider());
-    expectArrowNotSupported("TIMESTAMP_TZ", "2023-02-29T23:59:59.999999999Z");
-
-    // Test numeric strings
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ",
-        "0",
-        "1970-01-01 00:00:00.000000000 Z",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_TZ(0)",
+        "2022-01-31T13:00:00.123456+09:00",
+        "2022-01-31 13:00:00. +0900",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ",
-        "86399",
-        "1970-01-01 23:59:59.000000000 Z",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_TZ(1)",
+        "2022-01-31T13:00:00+09:00",
+        "2022-01-31 13:00:00.0 +0900",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ",
-        "86401",
-        "1970-01-02 00:00:01.000000000 Z",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_TZ(1)",
+        "2022-01-31T13:00:00.123456+09:00",
+        "2022-01-31 13:00:00.1 +0900",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ",
-        "-86401",
-        "1969-12-30 23:59:59.000000000 Z",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_TZ(2)",
+        "2022-01-31T13:00:00.1+09:00",
+        "2022-01-31 13:00:00.10 +0900",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ",
-        "-86399",
-        "1969-12-31 00:00:01.000000000 Z",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_TZ(2)",
+        "2022-01-31T13:00:00.123456+09:00",
+        "2022-01-31 13:00:00.12 +0900",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ",
-        "1662731080",
-        "2022-09-09 13:44:40.000000000 Z",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_TZ(8)",
+        "2022-01-31T13:00:00.1+09:00",
+        "2022-01-31 13:00:00.10000000 +0900",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ",
-        "1662731080123456789",
-        "2022-09-09 13:44:40.123456789 Z",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_TZ(8)",
+        "2022-01-31T13:00:00.123456789+09:00",
+        "2022-01-31 13:00:00.12345678 +0900",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_TZ(8)",
+        "2022-01-31T13:00:00.000000009+09:00",
+        "2022-01-31 13:00:00.00000000 +0900",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_TZ(8)",
+        "2022-01-31T13:00:00.000000019+09:00",
+        "2022-01-31 13:00:00.00000001 +0900",
         new StringProvider());
   }
 
   @Test
   public void testTimestampWithLocalTimeZone() throws Exception {
-    useLosAngelesTimeZone();
+    setJdbcSessionTimezone(TZ_LOS_ANGELES);
+    setChannelDefaultTimezone(TZ_LOS_ANGELES);
 
     // Test timestamp formats
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2/18/2008 02:36:48",
-        "2008-02-18 02:36:48.000000000 -0800",
+        "2013-04-28T20:57",
+        "2013-04-28 20:57:00.000000000 -0700",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28 20",
-        "2013-04-28 20:00:00.000000000 -0700",
+        "2013-04-28T20:57:01",
+        "2013-04-28 20:57:01.000000000 -0700",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28 20:57",
-        "2013-04-28 20:57:00.000000000 -0700",
+        "2013-04-28T20:57:01-07:00",
+        "2013-04-28 20:57:01.000000000 -0700",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28 20:57:01",
-        "2013-04-28 20:57:01.000000000 -0700",
+        "2013-04-28T20:57:01.123456",
+        "2013-04-28 20:57:01.123456000 -0700",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28 20:57:01 +07:00",
-        "2013-04-28 06:57:01.000000000 -0700",
+        "2013-04-28T20:57:01.123456789+07:00",
+        "2013-04-28 06:57:01.123456789 -0700",
         new StringProvider(),
         new StringProvider());
-    testJdbcTypeCompatibility(
+    // Here we test ingestion only because JDBC cannot ingest ISO_ZONED_DATE_TIME
+    testIngestion(
         "TIMESTAMP_LTZ",
-        "2013-04-28 20:57:01 +0700",
-        "2013-04-28 06:57:01.000000000 -0700",
-        new StringProvider(),
+        "2013-04-28T20:57:01.123456789+09:00[Asia/Tokyo]",
+        "2013-04-28 04:57:01.123456789 -0700",
         new StringProvider());
+
+    // Test date formats
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28 20:57:01-07",
-        "2013-04-28 20:57:01.000000000 -0700",
+        "2013-04-28",
+        "2013-04-28 00:00:00.000000000 -0700",
         new StringProvider(),
         new StringProvider());
+
+    // Test boundary time zone
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28 20:57:01-07:00",
-        "2013-04-28 20:57:01.000000000 -0700",
+        "2013-04-28T00:00:00+07:00",
+        "2013-04-27 10:00:00.000000000 -0700",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28 20:57:01.123456",
-        "2013-04-28 20:57:01.123456000 -0700",
+        "2013-04-28T00:00:00-07:00",
+        "2013-04-28 00:00:00.000000000 -0700",
         new StringProvider(),
         new StringProvider());
+
+    // Test leap years
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28 20:57:01.123456789 +07:00",
-        "2013-04-28 06:57:01.123456789 -0700",
+        "2024-02-29T23:59:59.999999999Z",
+        "2024-02-29 15:59:59.999999999 -0800",
         new StringProvider(),
         new StringProvider());
+    expectArrowNotSupported("TIMESTAMP_LTZ", "2023-02-29T23:59:59.999999999");
+
+    // Numeric strings
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28 20:57:01.123456789 +0700",
-        "2013-04-28 06:57:01.123456789 -0700",
+        "0",
+        "1969-12-31 16:00:00.000000000 -0800",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28 20:57:01.123456789+07",
-        "2013-04-28 06:57:01.123456789 -0700",
+        "1674478926",
+        "2023-01-23 05:02:06.000000000 -0800",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28 20:57:01.123456789+07:00",
-        "2013-04-28 06:57:01.123456789 -0700",
+        "1674478926123",
+        "2023-01-23 05:02:06.123000000 -0800",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28 20:57+07:00",
-        "2013-04-28 06:57:00.000000000 -0700",
+        "1674478926123456",
+        "2023-01-23 05:02:06.123456000 -0800",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28T20",
-        "2013-04-28 20:00:00.000000000 -0700",
+        "1674478926123456789",
+        "2023-01-23 05:02:06.123456789 -0800",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28T20:57",
-        "2013-04-28 20:57:00.000000000 -0700",
+        "-1674478926",
+        "1916-12-09 02:57:54.000000000 -0800",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28T20:57:01",
-        "2013-04-28 20:57:01.000000000 -0700",
+        "-1674478926123",
+        "1916-12-09 02:57:53.877000000 -0800",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28T20:57:01-07:00",
-        "2013-04-28 20:57:01.000000000 -0700",
+        "-1674478926123456",
+        "1916-12-09 02:57:53.876544000 -0800",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "2013-04-28T20:57:01.123456",
-        "2013-04-28 20:57:01.123456000 -0700",
+        "-1674478926123456789",
+        "1916-12-09 02:57:53.876543211 -0800",
         new StringProvider(),
         new StringProvider());
-    testJdbcTypeCompatibility(
+
+    // java.time.LocalDate
+    testIngestion(
         "TIMESTAMP_LTZ",
-        "2013-04-28T20:57:01.123456789+07:00",
-        "2013-04-28 06:57:01.123456789 -0700",
-        new StringProvider(),
+        LocalDate.parse("2007-12-03"),
+        "2007-12-03 00:00:00.000000000 -0800",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    // java.time.LocalDateTime
+    testIngestion(
         "TIMESTAMP_LTZ",
-        "2013-04-28T20:57+07:00",
-        "2013-04-28 06:57:00.000000000 -0700",
-        new StringProvider(),
+        LocalDateTime.parse("2007-12-03T10:15:30"),
+        "2007-12-03 10:15:30.000000000 -0800",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    // java.time.OffsetDateTime
+    testIngestion(
         "TIMESTAMP_LTZ",
-        "Mon Jul 08 18:09:51 +0000 2013",
-        "2013-07-08 11:09:51.000000000 -0700",
-        new StringProvider(),
+        OffsetDateTime.parse("2007-12-03T10:15:30+01:00"),
+        "2007-12-03 01:15:30.000000000 -0800",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    // java.time.ZonedDateTime
+    testIngestion(
         "TIMESTAMP_LTZ",
-        "Thu, 21 Dec 2000 04:01:07 PM",
-        "2000-12-21 16:01:07.000000000 -0800",
-        new StringProvider(),
+        ZonedDateTime.parse("2007-12-03T10:15:30.123456789+01:00[Europe/Paris]"),
+        "2007-12-03 01:15:30.123456789 -0800",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    // java.time.Instant
+    testIngestion(
         "TIMESTAMP_LTZ",
-        "Thu, 21 Dec 2000 04:01:07 PM +0200",
-        "2000-12-21 06:01:07.000000000 -0800",
-        new StringProvider(),
+        OffsetDateTime.parse("2007-12-03T10:15:30.123456789+00:00").toInstant(),
+        "2007-12-03 02:15:30.123456789 -0800",
         new StringProvider());
-    testJdbcTypeCompatibility(
+
+    // Verify that default timezone has impact on input without timezone information
+    setChannelDefaultTimezone(TZ_BERLIN);
+    setJdbcSessionTimezone(TZ_BERLIN);
+    testIngestion(
         "TIMESTAMP_LTZ",
-        "Thu, 21 Dec 2000 04:01:07.123456789 PM",
-        "2000-12-21 16:01:07.123456789 -0800",
-        new StringProvider(),
+        "1674478926123456789",
+        "2023-01-23 14:02:06.123456789 +0100",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    testIngestion(
         "TIMESTAMP_LTZ",
-        "Thu, 21 Dec 2000 04:01:07.123456789 PM +0200",
-        "2000-12-21 06:01:07.123456789 -0800",
-        new StringProvider(),
+        "2013-04-28T20:57:01.123456789",
+        "2013-04-28 20:57:01.123456789 +0200",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    testIngestion(
         "TIMESTAMP_LTZ",
-        "Thu, 21 Dec 2000 16:01:07",
-        "2000-12-21 16:01:07.000000000 -0800",
-        new StringProvider(),
+        LocalDate.parse("2007-12-03"),
+        "2007-12-03 00:00:00.000000000 +0100",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    testIngestion(
         "TIMESTAMP_LTZ",
-        "Thu, 21 Dec 2000 16:01:07 +0200",
-        "2000-12-21 06:01:07.000000000 -0800",
-        new StringProvider(),
+        LocalDateTime.parse("2007-12-03T10:15:30"),
+        "2007-12-03 10:15:30.000000000 +0100",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    testIngestion(
         "TIMESTAMP_LTZ",
-        "Thu, 21 Dec 2000 16:01:07.123456789",
-        "2000-12-21 16:01:07.123456789 -0800",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ",
-        "Thu, 21 Dec 2000 16:01:07.123456789 +0200",
-        "2000-12-21 06:01:07.123456789 -0800",
-        new StringProvider(),
-        new StringProvider());
-
-    // Test date formats
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ",
-        "2013-04-28",
-        "2013-04-28 00:00:00.000000000 -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ",
-        "17-DEC-1980",
-        "1980-12-17 00:00:00.000000000 -0800",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ",
-        "12/17/1980",
-        "1980-12-17 00:00:00.000000000 -0800",
-        new StringProvider(),
+        OffsetDateTime.parse("2007-12-03T10:15:30.123456789+00:00").toInstant(),
+        "2007-12-03 11:15:30.123456789 +0100",
         new StringProvider());
 
-    // Test boundary time zone
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ",
-        "2013-04-28T00:00:00+07:00",
-        "2013-04-27 10:00:00.000000000 -0700",
-        new StringProvider(),
+    // Limited scale
+    setChannelDefaultTimezone(TZ_TOKYO);
+    setJdbcSessionTimezone(TZ_TOKYO);
+    testIngestion(
+        "TIMESTAMP_LTZ(0)",
+        "2022-01-31T13:00:00+09:00",
+        "2022-01-31 13:00:00. +0900",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ",
-        "2013-04-28T00:00:00-07:00",
-        "2013-04-28 00:00:00.000000000 -0700",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_LTZ(0)",
+        "2022-01-31T13:00:00.123456+09:00",
+        "2022-01-31 13:00:00. +0900",
         new StringProvider());
-
-    // Test leap years
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ",
-        "2024-02-29T23:59:59.999999999Z",
-        "2024-02-29 15:59:59.999999999 -0800",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_LTZ(1)",
+        "2022-01-31T13:00:00+09:00",
+        "2022-01-31 13:00:00.0 +0900",
         new StringProvider());
-    expectArrowNotSupported("TIMESTAMP_LTZ", "2023-02-29T23:59:59.999999999Z");
-
-    // Test numeric strings
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ",
-        "0",
-        "1969-12-31 16:00:00.000000000 -0800",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_LTZ(1)",
+        "2022-01-31T13:00:00.123456+09:00",
+        "2022-01-31 13:00:00.1 +0900",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ",
-        "86399",
-        "1970-01-01 15:59:59.000000000 -0800",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_LTZ(2)",
+        "2022-01-31T13:00:00.1+09:00",
+        "2022-01-31 13:00:00.10 +0900",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ",
-        "86401",
-        "1970-01-01 16:00:01.000000000 -0800",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_LTZ(2)",
+        "2022-01-31T13:00:00.123456+09:00",
+        "2022-01-31 13:00:00.12 +0900",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ",
-        "-86401",
-        "1969-12-30 15:59:59.000000000 -0800",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_LTZ(8)",
+        "2022-01-31T13:00:00.1+09:00",
+        "2022-01-31 13:00:00.10000000 +0900",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ",
-        "-86399",
-        "1969-12-30 16:00:01.000000000 -0800",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_LTZ(8)",
+        "2022-01-31T13:00:00.123456789+09:00",
+        "2022-01-31 13:00:00.12345678 +0900",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ",
-        "1662731080",
-        "2022-09-09 06:44:40.000000000 -0700",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_LTZ(8)",
+        "2022-01-31T13:00:00.000000009+09:00",
+        "2022-01-31 13:00:00.00000000 +0900",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ",
-        "1662731080123456789",
-        "2022-09-09 06:44:40.123456789 -0700",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_LTZ(8)",
+        "2022-01-31T13:00:00.000000019+09:00",
+        "2022-01-31 13:00:00.00000001 +0900",
         new StringProvider());
   }
 
   @Test
   public void testTimestampWithoutTimeZone() throws Exception {
     // Test timestamp formats
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "2/18/2008 02:36:48",
-        "2008-02-18 02:36:48.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "2013-04-28 20",
-        "2013-04-28 20:00:00.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "2013-04-28 20:57",
-        "2013-04-28 20:57:00.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "2013-04-28 20:57:01",
-        "2013-04-28 20:57:01.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "2013-04-28 20:57:01 +07:00",
-        "2013-04-28 20:57:01.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "2013-04-28 20:57:01 +0700",
-        "2013-04-28 20:57:01.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "2013-04-28 20:57:01-07",
-        "2013-04-28 20:57:01.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "2013-04-28 20:57:01-07:00",
-        "2013-04-28 20:57:01.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "2013-04-28 20:57:01.123456",
-        "2013-04-28 20:57:01.123456000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "2013-04-28 20:57:01.123456789 +07:00",
-        "2013-04-28 20:57:01.123456789 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "2013-04-28 20:57:01.123456789 +0700",
-        "2013-04-28 20:57:01.123456789 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "2013-04-28 20:57:01.123456789+07",
-        "2013-04-28 20:57:01.123456789 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "2013-04-28 20:57:01.123456789+07:00",
-        "2013-04-28 20:57:01.123456789 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "2013-04-28 20:57+07:00",
-        "2013-04-28 20:57:00.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "2013-04-28T20",
-        "2013-04-28 20:00:00.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
     testJdbcTypeCompatibility(
         "TIMESTAMP_NTZ",
         "2013-04-28T20:57",
@@ -679,65 +551,11 @@ public void testTimestampWithoutTimeZone() throws Exception {
         "2013-04-28 20:57:01.123456789 Z",
         new StringProvider(),
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "2013-04-28T20:57+07:00",
-        "2013-04-28 20:57:00.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "Mon Jul 08 18:09:51 +0000 2013",
-        "2013-07-08 18:09:51.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "Thu, 21 Dec 2000 04:01:07 PM",
-        "2000-12-21 16:01:07.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "Thu, 21 Dec 2000 04:01:07 PM +0200",
-        "2000-12-21 16:01:07.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "Thu, 21 Dec 2000 04:01:07.123456789 PM",
-        "2000-12-21 16:01:07.123456789 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "Thu, 21 Dec 2000 04:01:07.123456789 PM +0200",
-        "2000-12-21 16:01:07.123456789 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "Thu, 21 Dec 2000 16:01:07",
-        "2000-12-21 16:01:07.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "Thu, 21 Dec 2000 16:01:07 +0200",
-        "2000-12-21 16:01:07.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "Thu, 21 Dec 2000 16:01:07.123456789",
-        "2000-12-21 16:01:07.123456789 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
+    // Here we test ingestion only because JDBC cannot ingest ISO_ZONED_DATE_TIME
+    testIngestion(
         "TIMESTAMP_NTZ",
-        "Thu, 21 Dec 2000 16:01:07.123456789 +0200",
-        "2000-12-21 16:01:07.123456789 Z",
-        new StringProvider(),
+        "2013-04-28T20:57:01.123456789+09:00[Asia/Tokyo]",
+        "2013-04-28 20:57:01.123456789 Z",
         new StringProvider());
 
     // Test date formats
@@ -747,18 +565,6 @@ public void testTimestampWithoutTimeZone() throws Exception {
         "2013-04-28 00:00:00.000000000 Z",
         new StringProvider(),
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "17-DEC-1980",
-        "1980-12-17 00:00:00.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "12/17/1980",
-        "1980-12-17 00:00:00.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
 
     // Test boundary time zone
     testJdbcTypeCompatibility(
@@ -781,9 +587,9 @@ public void testTimestampWithoutTimeZone() throws Exception {
         "2024-02-29 23:59:59.999999999 Z",
         new StringProvider(),
         new StringProvider());
-    expectArrowNotSupported("TIMESTAMP_NTZ", "2023-02-29T23:59:59.999999999Z");
+    expectArrowNotSupported("TIMESTAMP_NTZ", "2023-02-29T23:59:59.999999999");
 
-    // Test numeric strings
+    // Numeric strings
     testJdbcTypeCompatibility(
         "TIMESTAMP_NTZ",
         "0",
@@ -791,776 +597,266 @@ public void testTimestampWithoutTimeZone() throws Exception {
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "86399",
-        "1970-01-01 23:59:59.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "86401",
-        "1970-01-02 00:00:01.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "-86401",
-        "1969-12-30 23:59:59.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "-86399",
-        "1969-12-31 00:00:01.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "1662731080",
-        "2022-09-09 13:44:40.000000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ",
-        "1662731080123456789",
-        "2022-09-09 13:44:40.123456789 Z",
-        new StringProvider(),
-        new StringProvider());
-  }
-
-  @Test
-  public void testJavaTimeObjects() throws Exception {
-    // TIME (LocalTime and OffsetTime are supported)
-    testIngestion("TIME", LocalTime.of(23, 59, 59), "23:59:59.000000000 Z", new StringProvider());
-    testIngestion(
-        "TIME",
-        OffsetTime.of(23, 59, 59, 0, ZoneOffset.ofHoursMinutes(4, 0)),
-        "23:59:59.000000000 Z",
-        new StringProvider());
-    testIngestion(
-        "TIME",
-        OffsetTime.of(23, 59, 59, 0, ZoneOffset.ofHoursMinutes(-4, 0)),
-        "23:59:59.000000000 Z",
-        new StringProvider());
-    testIngestion(
-        "TIME",
-        OffsetTime.of(23, 59, 59, 123, ZoneOffset.ofHoursMinutes(-4, 0)),
-        "23:59:59.000000123 Z",
-        new StringProvider());
-    testIngestion(
-        "TIME",
-        OffsetTime.of(23, 59, 59, 123456789, ZoneOffset.ofHoursMinutes(-4, 0)),
-        "23:59:59.123456789 Z",
-        new StringProvider());
-
-    // DATE (LocalDate, LocalDateTime, OffsetDateTime, ZonedDateTime are supported)
-    testIngestion("DATE", LocalDate.parse("2007-12-03"), "2007-12-03", new StringProvider());
-    testIngestion(
-        "DATE", LocalDateTime.parse("2007-12-03T10:15:30"), "2007-12-03", new StringProvider());
-    testIngestion(
-        "DATE",
-        OffsetDateTime.parse("2007-12-03T10:15:30+01:00"),
-        "2007-12-03",
-        new StringProvider());
-    testIngestion(
-        "DATE",
-        OffsetDateTime.parse("2007-12-03T00:00:00+01:00"),
-        "2007-12-03",
-        new StringProvider());
-    testIngestion(
-        "DATE",
-        OffsetDateTime.parse("2007-12-03T00:00:00-08:00"),
-        "2007-12-03",
-        new StringProvider());
-    testIngestion(
-        "DATE",
-        ZonedDateTime.parse("2007-12-03T10:15:30+01:00[Europe/Paris]"),
-        "2007-12-03",
-        new StringProvider());
-    testIngestion(
-        "DATE",
-        ZonedDateTime.parse("2007-12-03T00:00:00+01:00[Europe/Paris]"),
-        "2007-12-03",
-        new StringProvider());
-    testIngestion(
-        "DATE",
-        ZonedDateTime.parse("2007-12-03T00:00:00-08:00[America/Los_Angeles]"),
-        "2007-12-03",
-        new StringProvider());
-    testIngestion(
-        "DATE",
-        ZonedDateTime.parse("2007-07-03T00:00:00-07:00[America/Los_Angeles]"),
-        "2007-07-03",
-        new StringProvider());
-    //
-    // TIMESTAMP_NTZ (LocalDate, LocalDateTime, OffsetDateTime, ZonedDateTime are supported)
-    testIngestion(
-        "TIMESTAMP_NTZ",
-        LocalDate.parse("2007-12-03"),
-        "2007-12-03 00:00:00.000000000 Z",
-        new StringProvider());
-    testIngestion(
-        "TIMESTAMP_NTZ",
-        LocalDateTime.parse("2007-12-03T10:15:30"),
-        "2007-12-03 10:15:30.000000000 Z",
-        new StringProvider());
-    testIngestion(
-        "TIMESTAMP_NTZ",
-        OffsetDateTime.parse("2007-12-03T10:15:30+01:00"),
-        "2007-12-03 10:15:30.000000000 Z",
-        new StringProvider());
-    testIngestion(
-        "TIMESTAMP_NTZ",
-        ZonedDateTime.parse("2007-12-03T10:15:30+01:00[Europe/Paris]"),
-        "2007-12-03 10:15:30.000000000 Z",
-        new StringProvider());
-    testIngestion(
-        "TIMESTAMP_NTZ",
-        ZonedDateTime.parse("2007-12-03T10:15:30.123456789+01:00[Europe/Paris]"),
-        "2007-12-03 10:15:30.123456789 Z",
-        new StringProvider());
-    testIngestion(
-        "TIMESTAMP_NTZ",
-        ZonedDateTime.parse("2007-07-03T10:15:30.123456789+02:00[Europe/Paris]"),
-        "2007-07-03 10:15:30.123456789 Z",
-        new StringProvider());
-
-    useLosAngelesTimeZone();
-    // TIMESTAMP_LTZ (LocalDate, LocalDateTime, OffsetDateTime, ZonedDateTime are supported)
-    testIngestion(
-        "TIMESTAMP_LTZ",
-        LocalDate.parse("2007-12-03"),
-        "2007-12-03 00:00:00.000000000 -0800",
-        new StringProvider());
-    testIngestion(
-        "TIMESTAMP_LTZ",
-        LocalDateTime.parse("2007-12-03T10:15:30"),
-        "2007-12-03 10:15:30.000000000 -0800",
-        new StringProvider());
-    testIngestion(
-        "TIMESTAMP_LTZ",
-        OffsetDateTime.parse("2007-12-03T10:15:30+01:00"),
-        "2007-12-03 01:15:30.000000000 -0800",
-        new StringProvider());
-    testIngestion(
-        "TIMESTAMP_LTZ",
-        ZonedDateTime.parse("2007-12-03T10:15:30+01:00[Europe/Paris]"),
-        "2007-12-03 01:15:30.000000000 -0800",
-        new StringProvider());
-    testIngestion(
-        "TIMESTAMP_LTZ",
-        ZonedDateTime.parse("2007-12-03T10:15:30.123456789+01:00[Europe/Paris]"),
-        "2007-12-03 01:15:30.123456789 -0800",
-        new StringProvider());
-    testIngestion(
-        "TIMESTAMP_LTZ",
-        ZonedDateTime.parse("2007-07-03T10:15:30.123456789+02:00[Europe/Paris]"),
-        "2007-07-03 01:15:30.123456789 -0700",
-        new StringProvider());
-
-    // TIMESTAMP_TZ (LocalDate, LocalDateTime, OffsetDateTime, ZonedDateTime are supported)
-    testIngestion(
-        "TIMESTAMP_TZ",
-        LocalDate.parse("2007-12-03"),
-        "2007-12-03 00:00:00.000000000 -0800",
-        new StringProvider());
-    testIngestion(
-        "TIMESTAMP_TZ",
-        LocalDateTime.parse("2007-12-03T10:15:30"),
-        "2007-12-03 10:15:30.000000000 -0800",
-        new StringProvider());
-    testIngestion(
-        "TIMESTAMP_TZ",
-        OffsetDateTime.parse("2007-12-03T10:15:30+01:00"),
-        "2007-12-03 10:15:30.000000000 +0100",
-        new StringProvider());
-    testIngestion(
-        "TIMESTAMP_TZ",
-        ZonedDateTime.parse("2007-12-03T10:15:30+01:00[Europe/Paris]"),
-        "2007-12-03 10:15:30.000000000 +0100",
-        new StringProvider());
-    testIngestion(
-        "TIMESTAMP_TZ",
-        ZonedDateTime.parse("2007-12-03T10:15:30.123456789+01:00[Europe/Paris]"),
-        "2007-12-03 10:15:30.123456789 +0100",
-        new StringProvider());
-    testIngestion(
-        "TIMESTAMP_TZ",
-        ZonedDateTime.parse("2007-07-03T10:15:30.123456789+02:00[Europe/Paris]"),
-        "2007-07-03 10:15:30.123456789 +0200",
-        new StringProvider());
-  }
-
-  @Test
-  public void testTime() throws Exception {
-    // All (7) documented time formats are supported
-    // https://docs.snowflake.com/en/user-guide/date-time-input-output.html#time-formats
-
-    testJdbcTypeCompatibility(
-        "TIME",
-        "20:57:01.123456789+07:00",
-        "20:57:01.123456789 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME",
-        "20:57:01.123456789",
-        "20:57:01.123456789 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME", "20:57:01", "20:57:01.000000000 Z", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME", "20:57", "20:57:00.000000000 Z", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME",
-        "07:57:01.123456789 AM",
-        "07:57:01.123456789 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME", "04:01:07 AM", "04:01:07.000000000 Z", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME", "04:01 PM", "16:01:00.000000000 Z", new StringProvider(), new StringProvider());
-
-    // Test numeric strings
-    testJdbcTypeCompatibility(
-        "TIME", "0", "00:00:00.000000000 Z", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME", "86399", "23:59:59.000000000 Z", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME", "86401", "00:00:01.000000000 Z", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME", "-86401", "23:59:59.000000000 Z", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME", "-86399", "00:00:01.000000000 Z", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME", "1662731080", "13:44:40.000000000 Z", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME",
-        "1662731080123456789",
-        "13:44:40.123456789 Z",
-        new StringProvider(),
-        new StringProvider());
-  }
-
-  @Test
-  public void testLimitedScale() throws Exception {
-    // Of TIME
-    testJdbcTypeCompatibility(
-        "TIME(0)", "13:00:00.999", "13:00:00. Z", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME(0)", "13:00:00.999999999", "13:00:00. Z", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME(0)",
-        "13:00:00.999999999+07:30",
-        "13:00:00. Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME(0)",
-        "1662731080123456789",
-        "13:44:40. Z",
-        new StringProvider(),
-        new StringProvider());
-
-    testJdbcTypeCompatibility(
-        "TIME(4)", "13:00:00.999", "13:00:00.9990 Z", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME(4)",
-        "13:00:00.999999999",
-        "13:00:00.9999 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME(4)",
-        "13:00:00.999999999+07:30",
-        "13:00:00.9999 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME(4)",
-        "1662731080123456789",
-        "13:44:40.1234 Z",
-        new StringProvider(),
-        new StringProvider());
-
-    testJdbcTypeCompatibility(
-        "TIME(9)",
-        "13:00:00.999",
-        "13:00:00.999000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME(9)",
-        "13:00:00.999999999",
-        "13:00:00.999999999 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME(9)",
-        "13:00:00.999999999+07:30",
-        "13:00:00.999999999 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME(9)",
-        "1662731080123456789",
-        "13:44:40.123456789 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIME(9)",
-        "1662731080123456",
-        "13:44:40.123456000 Z",
-        new StringProvider(),
-        new StringProvider());
-
-    // Of TIMESTAMP_NTZ
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(0)",
-        "2010-07-07 13:00:00.999",
-        "2010-07-07 13:00:00. Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(0)",
-        "2010-07-07 13:00:00.999999999",
-        "2010-07-07 13:00:00. Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(0)",
-        "2010-07-07 13:00:00.999999999+07:30",
-        "2010-07-07 13:00:00. Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(0)",
-        "1662731080123456789",
-        "2022-09-09 13:44:40. Z",
-        new StringProvider(),
-        new StringProvider());
-
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(4)",
-        "2010-07-07 13:00:00.999",
-        "2010-07-07 13:00:00.9990 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(4)",
-        "2010-07-07 13:00:00.999999999",
-        "2010-07-07 13:00:00.9999 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(4)",
-        "2010-07-07 13:00:00.999999999+07:30",
-        "2010-07-07 13:00:00.9999 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(4)",
-        "1662731080123456789",
-        "2022-09-09 13:44:40.1234 Z",
-        new StringProvider(),
-        new StringProvider());
-
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(7)",
-        "2010-07-07 13:00:00.999",
-        "2010-07-07 13:00:00.9990000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(7)",
-        "2010-07-07 13:00:00.999999999",
-        "2010-07-07 13:00:00.9999999 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(7)",
-        "2010-07-07 13:00:00.999999999+07:30",
-        "2010-07-07 13:00:00.9999999 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(7)",
-        "1662731080123456789",
-        "2022-09-09 13:44:40.1234567 Z",
-        new StringProvider(),
-        new StringProvider());
-
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(8)",
-        "2010-07-07 13:00:00.999",
-        "2010-07-07 13:00:00.99900000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(8)",
-        "2010-07-07 13:00:00.999999999",
-        "2010-07-07 13:00:00.99999999 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(8)",
-        "2010-07-07 13:00:00.999999999+07:30",
-        "2010-07-07 13:00:00.99999999 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(8)",
-        "1662731080123456789",
-        "2022-09-09 13:44:40.12345678 Z",
-        new StringProvider(),
-        new StringProvider());
-
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(9)",
-        "2010-07-07 13:00:00.999",
-        "2010-07-07 13:00:00.999000000 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(9)",
-        "2010-07-07 13:00:00.999999999",
-        "2010-07-07 13:00:00.999999999 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(9)",
-        "2010-07-07 13:00:00.999999999+07:30",
-        "2010-07-07 13:00:00.999999999 Z",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_NTZ(9)",
-        "1662731080123456789",
-        "2022-09-09 13:44:40.123456789 Z",
-        new StringProvider(),
-        new StringProvider());
-
-    // Of TIMESTAMP_LTZ
-    useLosAngelesTimeZone();
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(0)",
-        "2010-07-07 13:00:00.999",
-        "2010-07-07 13:00:00. -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(0)",
-        "2010-07-07 13:00:00.999999999",
-        "2010-07-07 13:00:00. -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(0)",
-        "2010-07-07 13:00:00.999999999+07:30",
-        "2010-07-06 22:30:00. -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(0)",
-        "1662731080123456789",
-        "2022-09-09 06:44:40. -0700",
-        new StringProvider(),
-        new StringProvider());
-
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(4)",
-        "2010-07-07 13:00:00.999",
-        "2010-07-07 13:00:00.9990 -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(4)",
-        "2010-07-07 13:00:00.999999999",
-        "2010-07-07 13:00:00.9999 -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(4)",
-        "2010-07-07 13:00:00.999999999+07:30",
-        "2010-07-06 22:30:00.9999 -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(4)",
-        "1662731080123456789",
-        "2022-09-09 06:44:40.1234 -0700",
-        new StringProvider(),
-        new StringProvider());
-
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(7)",
-        "2010-07-07 13:00:00.999",
-        "2010-07-07 13:00:00.9990000 -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(7)",
-        "2010-07-07 13:00:00.999999999",
-        "2010-07-07 13:00:00.9999999 -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(7)",
-        "2010-07-07 13:00:00.999999999+07:30",
-        "2010-07-06 22:30:00.9999999 -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(7)",
-        "1662731080123456789",
-        "2022-09-09 06:44:40.1234567 -0700",
-        new StringProvider(),
-        new StringProvider());
-
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(8)",
-        "2010-07-07 13:00:00.999",
-        "2010-07-07 13:00:00.99900000 -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(8)",
-        "2010-07-07 13:00:00.999999999",
-        "2010-07-07 13:00:00.99999999 -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(8)",
-        "2010-07-07 13:00:00.999999999+07:30",
-        "2010-07-06 22:30:00.99999999 -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(8)",
-        "1662731080123456789",
-        "2022-09-09 06:44:40.12345678 -0700",
-        new StringProvider(),
-        new StringProvider());
-
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(9)",
-        "2010-07-07 13:00:00.999",
-        "2010-07-07 13:00:00.999000000 -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(9)",
-        "2010-07-07 13:00:00.999999999",
-        "2010-07-07 13:00:00.999999999 -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(9)",
-        "2010-07-07 13:00:00.999999999+07:30",
-        "2010-07-06 22:30:00.999999999 -0700",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_LTZ(9)",
-        "1662731080123456789",
-        "2022-09-09 06:44:40.123456789 -0700",
-        new StringProvider(),
-        new StringProvider());
-
-    // Of TIMESTAMP_TZ
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(0)",
-        "2010-07-07 13:00:00.999",
-        "2010-07-07 13:00:00. -0700",
+        "TIMESTAMP_NTZ",
+        "1674478926",
+        "2023-01-23 13:02:06.000000000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(0)",
-        "2010-07-07 13:00:00.999999999",
-        "2010-07-07 13:00:00. -0700",
+        "TIMESTAMP_NTZ",
+        "1674478926123",
+        "2023-01-23 13:02:06.123000000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(0)",
-        "2010-07-07 13:00:00.999999999+07:30",
-        "2010-07-07 13:00:00. +0730",
+        "TIMESTAMP_NTZ",
+        "1674478926123456",
+        "2023-01-23 13:02:06.123456000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(0)",
-        "1662731080123456789",
-        "2022-09-09 13:44:40. Z",
+        "TIMESTAMP_NTZ",
+        "1674478926123456789",
+        "2023-01-23 13:02:06.123456789 Z",
         new StringProvider(),
         new StringProvider());
-
     testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(4)",
-        "2010-07-07 13:00:00.999",
-        "2010-07-07 13:00:00.9990 -0700",
+        "TIMESTAMP_NTZ",
+        "-1674478926",
+        "1916-12-09 10:57:54.000000000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(4)",
-        "2010-07-07 13:00:00.999999999",
-        "2010-07-07 13:00:00.9999 -0700",
+        "TIMESTAMP_NTZ",
+        "-1674478926123",
+        "1916-12-09 10:57:53.877000000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(4)",
-        "2010-07-07 13:00:00.999999999+07:30",
-        "2010-07-07 13:00:00.9999 +0730",
+        "TIMESTAMP_NTZ",
+        "-1674478926123456",
+        "1916-12-09 10:57:53.876544000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(4)",
-        "1662731080123456789",
-        "2022-09-09 13:44:40.1234 Z",
+        "TIMESTAMP_NTZ",
+        "-1674478926123456789",
+        "1916-12-09 10:57:53.876543211 Z",
         new StringProvider(),
         new StringProvider());
 
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(7)",
-        "2010-07-07 13:00:00.999",
-        "2010-07-07 13:00:00.9990000 -0700",
-        new StringProvider(),
+    // java.time.LocalDate
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        LocalDate.parse("2007-12-03"),
+        "2007-12-03 00:00:00.000000000 Z",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(7)",
-        "2010-07-07 13:00:00.999999999",
-        "2010-07-07 13:00:00.9999999 -0700",
-        new StringProvider(),
+    // java.time.LocalDateTime
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        LocalDateTime.parse("2007-12-03T10:15:30"),
+        "2007-12-03 10:15:30.000000000 Z",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(7)",
-        "2010-07-07 13:00:00.999999999+07:30",
-        "2010-07-07 13:00:00.9999999 +0730",
-        new StringProvider(),
+    // java.time.OffsetDateTime
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        OffsetDateTime.parse("2007-12-03T10:15:30+01:00"),
+        "2007-12-03 10:15:30.000000000 Z",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(7)",
-        "1662731080123456789",
-        "2022-09-09 13:44:40.1234567 Z",
-        new StringProvider(),
+    // java.time.ZonedDateTime
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        ZonedDateTime.parse("2007-12-03T10:15:30.123456789+01:00[Europe/Paris]"),
+        "2007-12-03 10:15:30.123456789 Z",
+        new StringProvider());
+    // java.time.Instant
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        OffsetDateTime.parse("2007-12-03T10:15:30.123456789+00:00").toInstant(),
+        "2007-12-03 10:15:30.123456789 Z",
         new StringProvider());
 
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(8)",
-        "2010-07-07 13:00:00.999",
-        "2010-07-07 13:00:00.99900000 -0700",
-        new StringProvider(),
+    // Verify that default timezone has no impact
+    setChannelDefaultTimezone(TZ_LOS_ANGELES);
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        "1674478926123456789",
+        "2023-01-23 13:02:06.123456789 Z",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(8)",
-        "2010-07-07 13:00:00.999999999",
-        "2010-07-07 13:00:00.99999999 -0700",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        "2013-04-28T20:57:01.123456789",
+        "2013-04-28 20:57:01.123456789 Z",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(8)",
-        "2010-07-07 13:00:00.999999999+07:30",
-        "2010-07-07 13:00:00.99999999 +0730",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        LocalDate.parse("2007-12-03"),
+        "2007-12-03 00:00:00.000000000 Z",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(8)",
-        "1662731080123456789",
-        "2022-09-09 13:44:40.12345678 Z",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        LocalDateTime.parse("2007-12-03T10:15:30"),
+        "2007-12-03 10:15:30.000000000 Z",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        OffsetDateTime.parse("2007-12-03T10:15:30.123456789+00:00").toInstant(),
+        "2007-12-03 10:15:30.123456789 Z",
         new StringProvider());
 
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(9)",
-        "2010-07-07 13:00:00.999",
-        "2010-07-07 13:00:00.999000000 -0700",
-        new StringProvider(),
+    // Limited scale
+    testIngestion(
+        "TIMESTAMP_NTZ(0)",
+        "2022-01-31T13:00:00+09:00",
+        "2022-01-31 13:00:00. Z",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(9)",
-        "2010-07-07 13:00:00.999999999",
-        "2010-07-07 13:00:00.999999999 -0700",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_NTZ(0)",
+        "2022-01-31T13:00:00.123456+09:00",
+        "2022-01-31 13:00:00. Z",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(9)",
-        "2010-07-07 13:00:00.999999999+07:30",
-        "2010-07-07 13:00:00.999999999 +0730",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_NTZ(1)",
+        "2022-01-31T13:00:00+09:00",
+        "2022-01-31 13:00:00.0 Z",
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "TIMESTAMP_TZ(9)",
-        "1662731080123456789",
-        "2022-09-09 13:44:40.123456789 Z",
-        new StringProvider(),
+    testIngestion(
+        "TIMESTAMP_NTZ(1)",
+        "2022-01-31T13:00:00.123456+09:00",
+        "2022-01-31 13:00:00.1 Z",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_NTZ(2)",
+        "2022-01-31T13:00:00.1+09:00",
+        "2022-01-31 13:00:00.10 Z",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_NTZ(2)",
+        "2022-01-31T13:00:00.123456+09:00",
+        "2022-01-31 13:00:00.12 Z",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_NTZ(8)",
+        "2022-01-31T13:00:00.1+09:00",
+        "2022-01-31 13:00:00.10000000 Z",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_NTZ(8)",
+        "2022-01-31T13:00:00.123456789+09:00",
+        "2022-01-31 13:00:00.12345678 Z",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_NTZ(8)",
+        "2022-01-31T13:00:00.000000009+09:00",
+        "2022-01-31 13:00:00.00000000 Z",
+        new StringProvider());
+    testIngestion(
+        "TIMESTAMP_NTZ(8)",
+        "2022-01-31T13:00:00.000000019+09:00",
+        "2022-01-31 13:00:00.00000001 Z",
         new StringProvider());
   }
 
   @Test
-  public void testDate() throws Exception {
-    // Test timestamp formats
-    testJdbcTypeCompatibility(
-        "DATE", "2/18/2008 02:36:48", "2008-02-18", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "DATE", "2013-04-28 20", "2013-04-28", new StringProvider(), new StringProvider());
+  public void testTime() throws Exception {
+    // Simple time parsing
     testJdbcTypeCompatibility(
-        "DATE", "2013-04-28 20:57", "2013-04-28", new StringProvider(), new StringProvider());
+        "TIME", "20:57", "20:57:00.000000000 Z", new StringProvider(), new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE", "2013-04-28 20:57:01", "2013-04-28", new StringProvider(), new StringProvider());
+        "TIME", "20:57:01", "20:57:01.000000000 Z", new StringProvider(), new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE",
-        "2013-04-28 20:57:01 +07:00",
-        "2013-04-28",
+        "TIME",
+        "20:57:01.123456789",
+        "20:57:01.123456789 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE",
-        "2013-04-28 20:57:01 +0700",
-        "2013-04-28",
+        "TIME",
+        "20:57:01.123456789+07:00",
+        "20:57:01.123456789 Z",
         new StringProvider(),
         new StringProvider());
+
+    // Numeric strings
+    testJdbcTypeCompatibility(
+        "TIME", "0", "00:00:00.000000000 Z", new StringProvider(), new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE", "2013-04-28 20:57:01-07", "2013-04-28", new StringProvider(), new StringProvider());
+        "TIME", "1674478926", "13:02:06.000000000 Z", new StringProvider(), new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE",
-        "2013-04-28 20:57:01-07:00",
-        "2013-04-28",
+        "TIME",
+        "1674478926123",
+        "13:02:06.123000000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE",
-        "2013-04-28 20:57:01.123456",
-        "2013-04-28",
+        "TIME",
+        "1674478926123456",
+        "13:02:06.123456000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE",
-        "2013-04-28 20:57:01.123456789 +07:00",
-        "2013-04-28",
+        "TIME",
+        "1674478926123456789",
+        "13:02:06.123456789 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE",
-        "2013-04-28 20:57:01.123456789 +0700",
-        "2013-04-28",
+        "TIME", "-1674478926", "10:57:54.000000000 Z", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIME",
+        "-1674478926123",
+        "10:57:53.877000000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE",
-        "2013-04-28 20:57:01.123456789+07",
-        "2013-04-28",
+        "TIME",
+        "-1674478926123456",
+        "10:57:53.876544000 Z",
         new StringProvider(),
         new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE",
-        "2013-04-28 20:57:01.123456789+07:00",
-        "2013-04-28",
+        "TIME",
+        "-1674478926123456789",
+        "10:57:53.876543211 Z",
         new StringProvider(),
         new StringProvider());
-    testJdbcTypeCompatibility(
-        "DATE", "2013-04-28 20:57+07:00", "2013-04-28", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "DATE", "2013-04-28T20", "2013-04-28", new StringProvider(), new StringProvider());
+
+    // java.time.LocalTime
+    testIngestion("TIME", LocalTime.of(23, 59, 59), "23:59:59.000000000 Z", new StringProvider());
+
+    // java.time.OffsetTime
+    testIngestion(
+        "TIME",
+        OffsetTime.of(23, 59, 59, 0, ZoneOffset.ofHoursMinutes(4, 0)),
+        "23:59:59.000000000 Z",
+        new StringProvider());
+    testIngestion(
+        "TIME",
+        OffsetTime.of(23, 59, 59, 123, ZoneOffset.ofHoursMinutes(-4, 0)),
+        "23:59:59.000000123 Z",
+        new StringProvider());
+    testIngestion(
+        "TIME",
+        OffsetTime.of(23, 59, 59, 123456789, ZoneOffset.ofHoursMinutes(-4, 0)),
+        "23:59:59.123456789 Z",
+        new StringProvider());
+
+    // Limited scale
+    testIngestion("TIME(0)", "13:00:00", "13:00:00. Z", new StringProvider());
+    testIngestion("TIME(0)", "13:00:00.123456", "13:00:00. Z", new StringProvider());
+    testIngestion("TIME(1)", "13:00:00", "13:00:00.0 Z", new StringProvider());
+    testIngestion("TIME(1)", "13:00:00.123456", "13:00:00.1 Z", new StringProvider());
+    testIngestion("TIME(2)", "13:00:00.1", "13:00:00.10 Z", new StringProvider());
+    testIngestion("TIME(2)", "13:00:00.123456", "13:00:00.12 Z", new StringProvider());
+    testIngestion("TIME(8)", "13:00:00.1", "13:00:00.10000000 Z", new StringProvider());
+    testIngestion("TIME(8)", "13:00:00.123456789", "13:00:00.12345678 Z", new StringProvider());
+    testIngestion("TIME(8)", "13:00:00.000000009", "13:00:00.00000000 Z", new StringProvider());
+    testIngestion("TIME(8)", "13:00:00.000000019", "13:00:00.00000001 Z", new StringProvider());
+  }
+
+  @Test
+  public void testDate() throws Exception {
+    // Test timestamp formats
     testJdbcTypeCompatibility(
         "DATE", "2013-04-28T20:57", "2013-04-28", new StringProvider(), new StringProvider());
     testJdbcTypeCompatibility(
@@ -1585,69 +881,45 @@ public void testDate() throws Exception {
         new StringProvider());
     testJdbcTypeCompatibility(
         "DATE", "2013-04-28T20:57+07:00", "2013-04-28", new StringProvider(), new StringProvider());
+
+    // Test date formats
     testJdbcTypeCompatibility(
+        "DATE", "2013-04-28", "2013-04-28", new StringProvider(), new StringProvider());
+
+    // Test LocalDate
+    testIngestion("DATE", LocalDate.parse("2007-12-03"), "2007-12-03", new StringProvider());
+    // Test LocalDateTime
+    testIngestion(
+        "DATE", LocalDateTime.parse("2007-12-03T10:15:30.123"), "2007-12-03", new StringProvider());
+    // Test OffsetDateTime
+    testIngestion(
         "DATE",
-        "Mon Jul 08 18:09:51 +0000 2013",
-        "2013-07-08",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "DATE",
-        "Thu, 21 Dec 2000 04:01:07 PM",
-        "2000-12-21",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "DATE",
-        "Thu, 21 Dec 2000 04:01:07 PM +0200",
-        "2000-12-21",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "DATE",
-        "Thu, 21 Dec 2000 04:01:07.123456789 PM",
-        "2000-12-21",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "DATE",
-        "Thu, 21 Dec 2000 04:01:07.123456789 PM +0200",
-        "2000-12-21",
-        new StringProvider(),
+        OffsetDateTime.parse("2007-12-03T02:15:30.123+09:00"),
+        "2007-12-03",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    testIngestion(
         "DATE",
-        "Thu, 21 Dec 2000 16:01:07",
-        "2000-12-21",
-        new StringProvider(),
+        OffsetDateTime.parse("2007-12-03T02:15:30.123-08:00"),
+        "2007-12-03",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    // Test ZonedDateTime
+    testIngestion(
         "DATE",
-        "Thu, 21 Dec 2000 16:01:07 +0200",
-        "2000-12-21",
-        new StringProvider(),
+        ZonedDateTime.parse("2007-12-03T02:15:30.123+09:00[Asia/Tokyo]"),
+        "2007-12-03",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    testIngestion(
         "DATE",
-        "Thu, 21 Dec 2000 16:01:07.123456789",
-        "2000-12-21",
-        new StringProvider(),
+        ZonedDateTime.parse("2007-12-03T02:15:30.123-08:00[America/Los_Angeles]"),
+        "2007-12-03",
         new StringProvider());
-    testJdbcTypeCompatibility(
+    // Test Instant
+    testIngestion(
         "DATE",
-        "Thu, 21 Dec 2000 16:01:07.123456789 +0200",
-        "2000-12-21",
-        new StringProvider(),
+        ZonedDateTime.parse("2007-12-03T02:15:30.123-08:00[America/Los_Angeles]").toInstant(),
+        "2007-12-03",
         new StringProvider());
 
-    // Test date formats
-    testJdbcTypeCompatibility(
-        "DATE", "2013-04-28", "2013-04-28", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "DATE", "17-DEC-1980", "1980-12-17", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "DATE", "12/17/1980", "1980-12-17", new StringProvider(), new StringProvider());
-
     // Test leap years
     testJdbcTypeCompatibility(
         "DATE",
@@ -1655,102 +927,111 @@ public void testDate() throws Exception {
         "2024-02-29",
         new StringProvider(),
         new StringProvider());
-    expectArrowNotSupported("DATE", "2023-02-29T23:59:59.999999999Z");
-
-    testJdbcTypeCompatibility(
-        "DATE", "9999-12-31", "9999-12-31", new StringProvider(), new StringProvider());
-    testJdbcTypeCompatibility(
-        "DATE",
-        "9999-12-31T23:59:59.999999999Z",
-        "9999-12-31",
-        new StringProvider(),
-        new StringProvider());
-
-    // Test boundary date
-    testJdbcTypeCompatibility(
-        "DATE",
-        "2013-04-28T00:00:00+07:00",
-        "2013-04-28",
-        new StringProvider(),
-        new StringProvider());
-    testJdbcTypeCompatibility(
-        "DATE",
-        "2013-04-28T00:00:00-07:00",
-        "2013-04-28",
-        new StringProvider(),
-        new StringProvider());
+    expectArrowNotSupported("DATE", "2023-02-29T23:59:59.999999999");
 
     // Test numeric strings
     testJdbcTypeCompatibility(
         "DATE", "0", "1970-01-01", new StringProvider(), new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE", "86399", "1970-01-01", new StringProvider(), new StringProvider());
+        "DATE", "1662731080", "2022-09-09", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "1662731080123", "2022-09-09", new StringProvider(), new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE", "86401", "1970-01-02", new StringProvider(), new StringProvider());
+        "DATE", "1662731080123456", "2022-09-09", new StringProvider(), new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE", "-86401", "1969-12-30", new StringProvider(), new StringProvider());
+        "DATE", "1662731080123456789", "2022-09-09", new StringProvider(), new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE", "-86399", "1969-12-31", new StringProvider(), new StringProvider());
+        "DATE", "-1674478926", "1916-12-09", new StringProvider(), new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE", "1662731080", "2022-09-09", new StringProvider(), new StringProvider());
+        "DATE", "-1674478926123", "1916-12-09", new StringProvider(), new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE", "1662731080123456789", "2022-09-09", new StringProvider(), new StringProvider());
+        "DATE", "-1674478926123456", "1916-12-09", new StringProvider(), new StringProvider());
+    testJdbcTypeCompatibility(
+        "DATE", "-1674478926123456789", "1916-12-09", new StringProvider(), new StringProvider());
   }
 
   @Test
-  @Ignore("SNOW-663646")
-  public void testOldValues() throws Exception {
+  public void testOldTimestamps() throws Exception {
+    testIngestion("DATE", "0001-12-31", "0001-12-31", new StringProvider());
+    testIngestion(
+        "TIMESTAMP_NTZ",
+        "0001-12-31T11:11:11",
+        "0001-12-31 11:11:11.000000000 Z",
+        new StringProvider());
+    // TODO uncomment once SNOW-727474 is resolved
+    //        testIngestion("TIMESTAMP_LTZ", "0001-12-31T11:11:11", "0001-12-31 03:11:11.000000000
+    // -0800", new StringProvider());
+    testIngestion(
+        "TIMESTAMP_TZ",
+        "0001-12-31T11:11:11+03:00",
+        "0001-12-31 11:11:11.000000000 +0300",
+        new StringProvider());
+  }
+
+  @Test
+  public void testJulianGregorianGap() throws Exception {
+    // During switch from Julian to Gregorian calendars, there is a 10-day gap. The next day after
+    // 1582-10-04 was 1582-10-15.
+    // Snowflake doesn't have any special handling of this, these dates can be normally inserted.
+    testJdbcTypeCompatibility(
+        "DATE", "1582-10-04", "1582-10-04", new StringProvider(), new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE", "1582-01-01", "1582-10-01", new StringProvider(), new StringProvider());
+        "DATE", "1582-10-08", "1582-10-08", new StringProvider(), new StringProvider());
     testJdbcTypeCompatibility(
-        "DATE", "1582-10-14", "1582-10-14", new StringProvider(), new StringProvider());
+        "DATE", "1582-10-15", "1582-10-15", new StringProvider(), new StringProvider());
   }
 
   @Test
   public void testFutureDates() throws Exception {
-    useLosAngelesTimeZone();
-
+    setJdbcSessionTimezone(TZ_LOS_ANGELES);
+    setChannelDefaultTimezone(TZ_LOS_ANGELES);
     testJdbcTypeCompatibility(
-        "DATE", "99999-12-31", "99999-12-31", new StringProvider(), new StringProvider());
+        "DATE", "9999-12-31", "9999-12-31", new StringProvider(), new StringProvider());
 
-    testJdbcTypeCompatibility( // SB16
+    // SB16
+    testJdbcTypeCompatibility(
         "TIMESTAMP_NTZ",
-        "9999-12-31 23:59:59.999999999",
+        "9999-12-31T23:59:59.999999999",
         "9999-12-31 23:59:59.999999999 Z",
         new StringProvider(),
         new StringProvider());
 
-    testJdbcTypeCompatibility( // SB8
+    // SB8
+    testJdbcTypeCompatibility(
         "TIMESTAMP_NTZ(7)",
-        "9999-12-31 23:59:59.999999999",
+        "9999-12-31T23:59:59.999999999",
         "9999-12-31 23:59:59.9999999 Z",
         new StringProvider(),
         new StringProvider());
 
-    testJdbcTypeCompatibility( // SB16
+    // SB16
+    testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ",
-        "9999-12-31 23:59:59.999999999",
+        "9999-12-31T23:59:59.999999999",
         "9999-12-31 23:59:59.999999999 -0800",
         new StringProvider(),
         new StringProvider());
 
-    testJdbcTypeCompatibility( // SB8
+    // SB8
+    testJdbcTypeCompatibility(
         "TIMESTAMP_LTZ(7)",
-        "9999-12-31 23:59:59.999999999",
+        "9999-12-31T23:59:59.999999999",
         "9999-12-31 23:59:59.9999999 -0800",
         new StringProvider(),
         new StringProvider());
 
+    // SB16
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
-        "9999-12-31 23:59:59.999999999",
+        "9999-12-31T23:59:59.999999999",
         "9999-12-31 23:59:59.999999999 -0800",
         new StringProvider(),
         new StringProvider());
 
+    // SB8
     testJdbcTypeCompatibility(
         "TIMESTAMP_TZ(3)",
-        "9999-12-31 23:59:59.999999999",
+        "9999-12-31T23:59:59.999999999",
         "9999-12-31 23:59:59.999 -0800",
         new StringProvider(),
         new StringProvider());
@@ -1760,7 +1041,7 @@ public void testFutureDates() throws Exception {
    * To make assertions of timestamps without timezone to work, make sure JDBC connection session
    * time zone is the same as the streaming ingest default timezone
    */
-  private void useLosAngelesTimeZone() throws SQLException {
-    conn.createStatement().execute("alter session set timezone = 'America/Los_Angeles';");
+  private void setJdbcSessionTimezone(ZoneId timezone) throws SQLException {
+    conn.createStatement().execute(String.format("alter session set timezone = '%s';", timezone));
   }
 }

From 0cb0fab98524daa8bdb5f0ab758ce6f994772bac Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 8 Feb 2023 11:10:37 -0800
Subject: [PATCH 124/356] NO-SNOW: various fixes/improvements in the SDK (#330)

This PR contains multiple fixes/improvements:

- Include client version in the failure event
- Add some basic verifications in the generated EP
- Add logging to debug a stats mismatch issue
- Add server side failure status code to client telemetry as well
---
 .../ingest/connection/TelemetryService.java   |  3 ++
 .../streaming/internal/AbstractRowBuffer.java |  1 +
 .../streaming/internal/ChannelCache.java      |  2 +-
 .../streaming/internal/ChannelData.java       | 12 ++++--
 .../ingest/streaming/internal/EpInfo.java     | 28 +++++++++++++
 .../streaming/internal/RowBufferStats.java    |  8 ++--
 ...nowflakeStreamingIngestClientInternal.java | 39 ++++++++++++-------
 .../net/snowflake/ingest/utils/Constants.java |  1 +
 .../streaming/internal/RowBufferTest.java     | 25 ++++++++++++
 9 files changed, 98 insertions(+), 21 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java
index ca0a9c41a..a330ff3c8 100644
--- a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java
+++ b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java
@@ -4,6 +4,8 @@
 
 package net.snowflake.ingest.connection;
 
+import static net.snowflake.ingest.connection.RequestBuilder.DEFAULT_VERSION;
+
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.Snapshot;
@@ -88,6 +90,7 @@ public void reportLatencyInSec(
   public void reportClientFailure(String summary, String exception) {
     ObjectNode msg = MAPPER.createObjectNode();
     msg.put("summary", summary);
+    msg.put("client_version", DEFAULT_VERSION);
     msg.put("exception", exception);
     send(TelemetryType.STREAMING_INGEST_CLIENT_FAILURE, msg);
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index c7fa76886..973b6e81b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -541,6 +541,7 @@ static EpInfo buildEpInfoFromStats(long rowCount, Map<String, RowBufferStats> co
       String colName = colStat.getValue().getColumnDisplayName();
       epInfo.getColumnEps().put(colName, dto);
     }
+    epInfo.verifyEpInfo();
     return epInfo;
   }
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
index ef366b120..2cfeefc00 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
@@ -20,7 +20,7 @@
 class ChannelCache<T> {
   // Cache to hold all the valid channels, the key for the outer map is FullyQualifiedTableName and
   // the key for the inner map is ChannelName
-  private ConcurrentHashMap<
+  private final ConcurrentHashMap<
           String, ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<T>>>
       cache = new ConcurrentHashMap<>();
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
index ffc327d9d..91f2a2a68 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
@@ -43,11 +43,17 @@ public static Map<String, RowBufferStats> getCombinedColumnStatsMap(
     if (left == null || right == null) {
       throw new SFException(ErrorCode.INTERNAL_ERROR, "null column stats");
     }
+
     if (left.size() != right.size()) {
-      throw new SFException(ErrorCode.INTERNAL_ERROR, "Column stats map key mismatch");
+      throw new SFException(
+          ErrorCode.INTERNAL_ERROR,
+          String.format(
+              "Column stats map key size mismatch, left=%d, right=%d, leftKeySet=%s,"
+                  + " rightKeySet=%s",
+              left.size(), right.size(), left.keySet(), right.keySet()));
     }
-    Map<String, RowBufferStats> result = new HashMap<>();
 
+    Map<String, RowBufferStats> result = new HashMap<>();
     try {
       for (String key : left.keySet()) {
         RowBufferStats leftStats = left.get(key);
@@ -55,7 +61,7 @@ public static Map<String, RowBufferStats> getCombinedColumnStatsMap(
         result.put(key, RowBufferStats.getCombinedStats(leftStats, rightStats));
       }
     } catch (NullPointerException npe) {
-      throw new SFException(ErrorCode.INTERNAL_ERROR, "Column stats map key mismatch");
+      throw new SFException(npe, ErrorCode.INTERNAL_ERROR, "Column stats map key mismatch");
     }
     return result;
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/EpInfo.java b/src/main/java/net/snowflake/ingest/streaming/internal/EpInfo.java
index 32f77ceac..e6e6a4d9d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/EpInfo.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/EpInfo.java
@@ -1,7 +1,11 @@
 package net.snowflake.ingest.streaming.internal;
 
+import static net.snowflake.ingest.utils.Constants.EP_NDV_UNKNOWN;
+
 import com.fasterxml.jackson.annotation.JsonProperty;
 import java.util.Map;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.SFException;
 
 /** Class used to serialize/deserialize EP information */
 class EpInfo {
@@ -17,6 +21,30 @@ class EpInfo {
     this.columnEps = columnEps;
   }
 
+  /** Some basic verification logic to make sure the EP info is correct */
+  public void verifyEpInfo() {
+    for (Map.Entry<String, FileColumnProperties> entry : columnEps.entrySet()) {
+      String colName = entry.getKey();
+      FileColumnProperties colEp = entry.getValue();
+      // Make sure the null count should always smaller than the total row count
+      if (colEp.getNullCount() > rowCount) {
+        throw new SFException(
+            ErrorCode.INTERNAL_ERROR,
+            String.format(
+                "Null count bigger than total row count on col=%s, nullCount=%s, rowCount=%s",
+                colName, colEp.getNullCount(), rowCount));
+      }
+
+      // Make sure the NDV should always be -1
+      if (colEp.getDistinctValues() != EP_NDV_UNKNOWN) {
+        throw new SFException(
+            ErrorCode.INTERNAL_ERROR,
+            String.format(
+                "Unexpected NDV on col=%s, value=%d", colName, colEp.getDistinctValues()));
+      }
+    }
+  }
+
   @JsonProperty("rows")
   long getRowCount() {
     return rowCount;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
index 7be1c0050..06b3b3766 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
@@ -4,6 +4,8 @@
 
 package net.snowflake.ingest.streaming.internal;
 
+import static net.snowflake.ingest.utils.Constants.EP_NDV_UNKNOWN;
+
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
 import java.util.Objects;
@@ -22,12 +24,10 @@ class RowBufferStats {
   private long currentNullCount;
   // for binary or string columns
   private long currentMaxLength;
-  private String collationDefinitionString;
+  private final String collationDefinitionString;
   /** Display name is required for the registration endpoint */
   private final String columnDisplayName;
 
-  private static final int MAX_LOB_LEN = 32;
-
   /** Creates empty stats */
   RowBufferStats(String columnDisplayName, String collationDefinitionString) {
     this.columnDisplayName = columnDisplayName;
@@ -193,7 +193,7 @@ long getCurrentMaxLength() {
    * @return -1 indicating the NDV is unknown
    */
   long getDistinctValues() {
-    return -1;
+    return EP_NDV_UNKNOWN;
   }
 
   String getCollationDefinitionString() {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 8e319f408..4ad2c96ec 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -383,11 +383,17 @@ ChannelsStatusResponse getChannelsStatus(
         ChannelsStatusResponse.ChannelStatusResponseDTO channelStatus =
             response.getChannels().get(idx);
         if (channelStatus.getStatusCode() != RESPONSE_SUCCESS) {
-          logger.logWarn(
-              "Channel has failure status_code, name={}, channel_sequencer={}," + " status_code={}",
-              channel.getFullyQualifiedName(),
-              channel.getChannelSequencer(),
-              channelStatus.getStatusCode());
+          String errorMessage =
+              String.format(
+                  "Channel has failure status_code, name=%s, channel_sequencer=%d, status_code=%d",
+                  channel.getFullyQualifiedName(),
+                  channel.getChannelSequencer(),
+                  channelStatus.getStatusCode());
+          logger.logWarn(errorMessage);
+          if (getTelemetryService() != null) {
+            getTelemetryService()
+                .reportClientFailure(this.getClass().getSimpleName(), errorMessage);
+          }
         }
       }
 
@@ -481,14 +487,21 @@ void registerBlobs(List<BlobMetadata> blobs, final int executionCount) {
                                                 < MAX_STREAMING_INGEST_API_CHANNEL_RETRY) {
                                           queueFullChunks.add(chunkStatus);
                                         } else {
-                                          logger.logWarn(
-                                              "Channel has been invalidated because of failure"
-                                                  + " response, name={}, channel_sequencer={},"
-                                                  + " status_code={}, executionCount={}",
-                                              channelStatus.getChannelName(),
-                                              channelStatus.getChannelSequencer(),
-                                              channelStatus.getStatusCode(),
-                                              executionCount);
+                                          String errorMessage =
+                                              String.format(
+                                                  "Channel has been invalidated because of failure"
+                                                      + " response, name=%s, channel_sequencer=%d,"
+                                                      + " status_code=%d, executionCount=%d",
+                                                  channelStatus.getChannelName(),
+                                                  channelStatus.getChannelSequencer(),
+                                                  channelStatus.getStatusCode(),
+                                                  executionCount);
+                                          logger.logWarn(errorMessage);
+                                          if (getTelemetryService() != null) {
+                                            getTelemetryService()
+                                                .reportClientFailure(
+                                                    this.getClass().getSimpleName(), errorMessage);
+                                          }
                                           channelCache.invalidateChannelIfSequencersMatch(
                                               chunkStatus.getDBName(),
                                               chunkStatus.getSchemaName(),
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index 7d2903d60..13c3c5c3e 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -52,6 +52,7 @@ public class Constants {
   public static final int MAX_STREAMING_INGEST_API_CHANNEL_RETRY = 3;
   public static final int STREAMING_INGEST_TELEMETRY_UPLOAD_INTERVAL_IN_SEC = 10;
   public static final int LOW_RUNTIME_MEMORY_THRESHOLD_IN_BYTES = 100 * 1024 * 1024;
+  public static final long EP_NDV_UNKNOWN = -1L;
 
   // Channel level constants
   public static final String CHANNEL_STATUS_ENDPOINT = "/v1/streaming/channels/status/";
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 00e92c76b..41aa793d3 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -577,6 +577,31 @@ public void testBuildEpInfoFromNullColumnStats() {
     Assert.assertEquals(0, realColumnResult.getMaxLength());
   }
 
+  @Test
+  public void testInvalidEPInfo() {
+    Map<String, RowBufferStats> colStats = new HashMap<>();
+
+    RowBufferStats stats1 = new RowBufferStats("intColumn");
+    stats1.addIntValue(BigInteger.valueOf(2));
+    stats1.addIntValue(BigInteger.valueOf(10));
+    stats1.addIntValue(BigInteger.valueOf(1));
+
+    RowBufferStats stats2 = new RowBufferStats("strColumn");
+    stats2.addStrValue("alice");
+    stats2.incCurrentNullCount();
+    stats2.incCurrentNullCount();
+
+    colStats.put("intColumn", stats1);
+    colStats.put("strColumn", stats2);
+
+    try {
+      AbstractRowBuffer.buildEpInfoFromStats(1, colStats);
+      Assert.fail("should fail when row count is smaller than null count.");
+    } catch (SFException e) {
+      Assert.assertEquals(ErrorCode.INTERNAL_ERROR.getMessageCode(), e.getVendorCode());
+    }
+  }
+
   @Test
   public void testE2E() {
     testE2EHelper(this.rowBufferOnErrorAbort);

From e6e65e49a638bf6ed1594419d7a13df204f774c4 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 8 Feb 2023 12:12:48 -0800
Subject: [PATCH 125/356] V1.0.3-beta release (#326)

This release contains a few bug fixes and improvements for Snowpipe Streaming:

[Improvement] Add column name to data validation error messages
[Improvement] Convert empty variant strings into variant column to NULL
[Improvement] Avoid datetime parsing wherever possible to improve performance
[Improvement] Allow inf, -inf and nan strings into FLOAT columns
[Improvement] Add telemetry to track the end2end latency
[Fixes] Fix small parquet binary max length inconsistency with Arrow
[Improvement] Add option to use parquet file writer per channel and then merge them into one per chunk
[Improvement] Improve build latency for high throughput case with many channels
[Fix] Fix various issues with unicode strings
[Fix] Ingest SDK Does Not Honor http.nonProxyHosts JVM Argument
[Improvement] Enable Parquet file by default and enforce single rowcount Parquet files
This release contains a few bug fixes and improvements for Snowpipe:

[Fix] SimpleIngestIT runtime created database
---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 4fec9dbc4..8eb91d100 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
     <!-- Arifact name and version information -->
     <groupId>net.snowflake</groupId>
     <artifactId>snowflake-ingest-sdk</artifactId>
-    <version>1.0.2-beta.8</version>
+    <version>1.0.3-beta</version>
     <packaging>jar</packaging>
     <name>Snowflake Ingest SDK</name>
     <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 3588883d4..16f0b8d9d 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -127,7 +127,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.0.2-beta.8";
+  public static final String DEFAULT_VERSION = "1.0.3-beta";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 26ce7d64944b81632bb9b4bc2ecb95e216b1b20c Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Wed, 8 Feb 2023 23:01:27 +0100
Subject: [PATCH 126/356] SNOW-663704 Send hex-encoded min/max for BINARY
 (#329)

Instead of creating strings from binary arrays, we will now send them
hex-encoded to the server side. The server-side is ready to accept
hex-encoded min/max values since the release 6.36.
---
 .github/workflows/End2EndTest.yml             |  2 +-
 .../streaming/internal/ArrowRowBuffer.java    |  3 +-
 .../streaming/internal/BinaryStringUtils.java |  8 ++-
 .../internal/ParquetValueParser.java          |  6 +-
 .../streaming/internal/RowBufferStats.java    | 12 ++--
 .../datatypes/AbstractDataTypeTest.java       | 19 +++++++
 .../internal/datatypes/BinaryIT.java          | 57 ++++++++++++++++---
 .../internal/datatypes/StringsIT.java         | 30 +++-------
 8 files changed, 91 insertions(+), 46 deletions(-)

diff --git a/.github/workflows/End2EndTest.yml b/.github/workflows/End2EndTest.yml
index 5f04e595f..d5307e6e8 100644
--- a/.github/workflows/End2EndTest.yml
+++ b/.github/workflows/End2EndTest.yml
@@ -8,7 +8,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         java: [ 8 ]
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 9e8b2880e..1adc14eda 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -7,7 +7,6 @@
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.math.RoundingMode;
-import java.nio.charset.StandardCharsets;
 import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -692,7 +691,7 @@ private float convertRowToArrow(
                     value,
                     Optional.ofNullable(maxLengthString).map(Integer::parseInt));
             ((VarBinaryVector) vector).setSafe(curRowIndex, bytes);
-            stats.addStrValue(new String(bytes, StandardCharsets.UTF_8));
+            stats.addBinaryValue(bytes);
             rowBufferSize += bytes.length;
             break;
           case REAL:
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BinaryStringUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/BinaryStringUtils.java
index a518747b1..21abed8f5 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BinaryStringUtils.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BinaryStringUtils.java
@@ -20,12 +20,16 @@ static String truncateBytesAsHex(byte[] bytes, boolean truncateUp) {
       return Hex.encodeHexString(bytes);
     }
 
+    // In order not to mutate the original byte array, let's make a copy
+    byte[] result = new byte[MAX_LOB_LEN];
+    System.arraycopy(bytes, 0, result, 0, MAX_LOB_LEN);
+
     // Round the least significant byte(s) up
     if (truncateUp) {
       int idx;
       for (idx = MAX_LOB_LEN - 1; idx >= 0; idx--) {
         // increment the current byte, if there was no overflow, we can stop
-        if (++bytes[idx] != 0) {
+        if (++result[idx] != 0) {
           break;
         }
       }
@@ -35,6 +39,6 @@ static String truncateBytesAsHex(byte[] bytes, boolean truncateUp) {
       }
     }
 
-    return Hex.encodeHexString(ByteBuffer.wrap(bytes, 0, MAX_LOB_LEN));
+    return Hex.encodeHexString(ByteBuffer.wrap(result, 0, MAX_LOB_LEN));
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
index 0f0442211..3ac5b80ca 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
@@ -7,7 +7,6 @@
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.math.RoundingMode;
-import java.nio.charset.StandardCharsets;
 import java.time.ZoneId;
 import java.util.Optional;
 import javax.annotation.Nullable;
@@ -344,10 +343,7 @@ private static byte[] getBinaryValueForLogicalBinary(
     byte[] bytes =
         DataValidationUtil.validateAndParseBinary(
             columnMetadata.getName(), value, Optional.of(maxLengthString).map(Integer::parseInt));
-
-    String str = new String(bytes, StandardCharsets.UTF_8);
-    stats.addStrValue(str);
-
+    stats.addBinaryValue(bytes);
     return bytes;
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
index 06b3b3766..66848742b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
@@ -74,13 +74,13 @@ static RowBufferStats getCombinedStats(RowBufferStats left, RowBufferStats right
     }
 
     if (left.currentMinStrValue != null) {
-      combined.addStrValue(left.currentMinStrValue);
-      combined.addStrValue(left.currentMaxStrValue);
+      combined.addBinaryValue(left.currentMinStrValue);
+      combined.addBinaryValue(left.currentMaxStrValue);
     }
 
     if (right.currentMinStrValue != null) {
-      combined.addStrValue(right.currentMinStrValue);
-      combined.addStrValue(right.currentMaxStrValue);
+      combined.addBinaryValue(right.currentMinStrValue);
+      combined.addBinaryValue(right.currentMaxStrValue);
     }
 
     if (left.currentMinRealValue != null) {
@@ -100,10 +100,10 @@ static RowBufferStats getCombinedStats(RowBufferStats left, RowBufferStats right
   }
 
   void addStrValue(String value) {
-    addStrValue(value.getBytes(StandardCharsets.UTF_8));
+    addBinaryValue(value.getBytes(StandardCharsets.UTF_8));
   }
 
-  void addStrValue(byte[] valueBytes) {
+  void addBinaryValue(byte[] valueBytes) {
     this.setCurrentMaxLength(valueBytes.length);
 
     // Check if new min/max string
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index f3b575527..49ab8b023 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -353,4 +353,23 @@ <STREAMING_INGEST_WRITE> void assertVariant(
   protected void migrateTable(String tableName) throws SQLException {
     conn.createStatement().execute(String.format("alter table %s migrate;", tableName));
   }
+
+  /**
+   * Ingest multiple values, wait for the latest offset to be committed, migrate the table and
+   * assert no errors have been thrown.
+   */
+  @SafeVarargs
+  protected final <STREAMING_INGEST_WRITE> void ingestManyAndMigrate(
+      String datatype, final STREAMING_INGEST_WRITE... values) throws Exception {
+    String tableName = createTable(datatype);
+    SnowflakeStreamingIngestChannel channel = openChannel(tableName);
+    String offsetToken = null;
+    for (int i = 0; i < values.length; i++) {
+      offsetToken = String.format("offsetToken%d", i);
+      channel.insertRow(createStreamingIngestRow(values[i]), offsetToken);
+    }
+
+    TestUtils.waitForOffset(channel, offsetToken);
+    migrateTable(tableName); // migration should always succeed
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
index a7b9d6cd7..d79d6836b 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
@@ -1,7 +1,7 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
+import net.snowflake.client.jdbc.internal.org.bouncycastle.util.encoders.Hex;
 import net.snowflake.ingest.utils.Constants;
-import org.junit.Ignore;
 import org.junit.Test;
 
 public class BinaryIT extends AbstractDataTypeTest {
@@ -11,13 +11,38 @@ public BinaryIT(String name, Constants.BdecVersion bdecVersion) {
   }
 
   @Test
-  public void testBinarySimple() throws Exception {
+  public void testBinary() throws Exception {
     testJdbcTypeCompatibility("BINARY", new byte[0], new ByteArrayProvider());
     testJdbcTypeCompatibility("BINARY", new byte[3], new ByteArrayProvider());
+    testJdbcTypeCompatibility(
+        "BINARY", new byte[] {Byte.MIN_VALUE, 0, Byte.MAX_VALUE}, new ByteArrayProvider());
+    testJdbcTypeCompatibility("BINARY", Hex.decode("a0b0c0d0e0f0"), new ByteArrayProvider());
+    testJdbcTypeCompatibility("BINARY", Hex.decode("aaff"), new ByteArrayProvider());
+    testJdbcTypeCompatibility(
+        "BINARY",
+        Hex.decode("0000000000000000000000000000000000000000000000000000000000000000"),
+        new ByteArrayProvider());
+    testJdbcTypeCompatibility(
+        "BINARY",
+        Hex.decode("000000000000000000000000000000000000000000000000000000000000000000"),
+        new ByteArrayProvider());
+    testJdbcTypeCompatibility(
+        "BINARY",
+        Hex.decode("aaffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
+        new ByteArrayProvider());
+    testJdbcTypeCompatibility(
+        "BINARY",
+        Hex.decode("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
+        new ByteArrayProvider());
+    testJdbcTypeCompatibility(
+        "BINARY",
+        Hex.decode("aaffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
+        new ByteArrayProvider());
+    testJdbcTypeCompatibility(
+        "BINARY",
+        Hex.decode("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
+        new ByteArrayProvider());
 
-    testJdbcTypeCompatibility("BINARY", new byte[8 * 1024 * 1024], new ByteArrayProvider());
-
-    testJdbcTypeCompatibility("BINARY", new byte[] {1, 2, 3, 4}, new ByteArrayProvider());
     testJdbcTypeCompatibility(
         "BINARY", "212D", new byte[] {33, 45}, new StringProvider(), new ByteArrayProvider());
     testJdbcTypeCompatibility(
@@ -29,8 +54,24 @@ public void testBinarySimple() throws Exception {
   }
 
   @Test
-  @Ignore("SNOW-663704")
-  public void testBinary() throws Exception {
-    testJdbcTypeCompatibility("BINARY", new byte[] {-1}, new ByteArrayProvider());
+  public void testBinaryComparison() throws Exception {
+    ingestManyAndMigrate(
+        "BINARY", new byte[] {Byte.MIN_VALUE}, new byte[] {Byte.MAX_VALUE}, new byte[] {0});
+    ingestManyAndMigrate(
+        "BINARY", new byte[] {Byte.MAX_VALUE}, new byte[] {Byte.MIN_VALUE}, new byte[] {0});
+    ingestManyAndMigrate(
+        "BINARY",
+        Hex.decode("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
+        Hex.decode("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00"));
+    ingestManyAndMigrate(
+        "BINARY",
+        Hex.decode("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"),
+        Hex.decode("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00"));
+  }
+
+  @Test
+  public void testMaxBinary() throws Exception {
+    byte[] arr = new byte[8 * 1024 * 1024];
+    testJdbcTypeCompatibility("BINARY", arr, new ByteArrayProvider());
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
index 4f5ac6e78..2af1f5803 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
@@ -6,8 +6,6 @@
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
 import java.sql.SQLException;
-import net.snowflake.ingest.TestUtils;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
@@ -131,30 +129,36 @@ public void testPrefixFF() throws Exception {
 
     // chars + 15+ times \uFFFF
     ingestManyAndMigrate(
+        "VARCHAR",
         "aaaaaaaaa\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF");
 
     // chars + 15+ times \uFFFF + chars
     ingestManyAndMigrate(
+        "VARCHAR",
         "aaaaaaaaa\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFFaaaaaaaaa");
 
     // 15+ times \uFFFF
     ingestManyAndMigrate(
+        "VARCHAR",
         "\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF");
 
     // 15+ times \uFFFF + chars
     ingestManyAndMigrate(
+        "VARCHAR",
         "\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFFaaaaaaaaa");
   }
 
   @Test
   public void testMultiByteCharComparison() throws Exception {
-    ingestManyAndMigrate("a", "❄");
-    ingestManyAndMigrate("❄", "a");
+    ingestManyAndMigrate("VARCHAR", "a", "❄");
+    ingestManyAndMigrate("VARCHAR", "❄", "a");
 
     ingestManyAndMigrate(
+        "VARCHAR",
         "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
         "❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄");
     ingestManyAndMigrate(
+        "VARCHAR",
         "❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄❄",
         "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
   }
@@ -215,22 +219,4 @@ public void testCollatedColumnsNotSupported() throws SQLException {
       Assert.assertEquals(ErrorCode.UNSUPPORTED_DATA_TYPE.getMessageCode(), e.getVendorCode());
     }
   }
-
-  /**
-   * Ingest multiple values, wait for the latest offset to be committed, migrate the table and
-   * assert no errors have been thrown.
-   */
-  protected <STREAMING_INGEST_WRITE> void ingestManyAndMigrate(STREAMING_INGEST_WRITE... values)
-      throws Exception {
-    String tableName = createTable("VARCHAR");
-    SnowflakeStreamingIngestChannel channel = openChannel(tableName);
-    String offsetToken = null;
-    for (int i = 0; i < values.length; i++) {
-      offsetToken = String.format("offsetToken%d", i);
-      channel.insertRow(createStreamingIngestRow(values[i]), offsetToken);
-    }
-
-    TestUtils.waitForOffset(channel, offsetToken);
-    migrateTable(tableName); // migration should always succeed
-  }
 }

From e1027faa57586f6bb9b92ba0668b3ca8d771f58c Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Thu, 16 Feb 2023 15:54:38 +0100
Subject: [PATCH 127/356] Remove JAXB dependency (#332)

This PR remove the dependency on `javax.xml.bind` that we used for hex encoding/decoding. We are going to use commons-codec instead.

PR is based on #328.
---
 pom.xml                                                  | 9 ---------
 .../snowflake/ingest/streaming/internal/BlobBuilder.java | 4 ++--
 .../ingest/streaming/internal/DataValidationUtil.java    | 7 ++++---
 .../streaming/internal/DataValidationUtilTest.java       | 8 ++++----
 4 files changed, 10 insertions(+), 18 deletions(-)

diff --git a/pom.xml b/pom.xml
index 8eb91d100..13db98906 100644
--- a/pom.xml
+++ b/pom.xml
@@ -227,15 +227,6 @@
             <scope>test</scope>
         </dependency>
 
-        <!-- java.lang.NoClassDefFoundError: javax/xml/bind/JAXBException -->
-        <!-- https://stackoverflow.com/questions/43574426/how-to-resolve-java
-        -lang-noclassdeffounderror-javax-xml-bind-jaxbexception-in-j/48404582-->
-        <dependency>
-            <groupId>javax.xml.bind</groupId>
-            <artifactId>jaxb-api</artifactId>
-            <version>2.3.1</version>
-        </dependency>
-
         <!-- JUnit so that we can make some basic unit tests -->
         <dependency>
             <groupId>junit</groupId>
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
index 9e34b30fd..4249130bb 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
@@ -29,11 +29,11 @@
 import javax.crypto.BadPaddingException;
 import javax.crypto.IllegalBlockSizeException;
 import javax.crypto.NoSuchPaddingException;
-import javax.xml.bind.DatatypeConverter;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.Cryptor;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.Pair;
+import org.apache.commons.codec.binary.Hex;
 
 /**
  * Build a single blob file that contains file header plus data. The header will be a
@@ -319,7 +319,7 @@ static String computeMD5(byte[] data, int length) throws NoSuchAlgorithmExceptio
         MessageDigest.getInstance("MD5");
     md.update(data, 0, length);
     byte[] digest = md.digest();
-    return DatatypeConverter.printHexBinary(digest).toLowerCase();
+    return Hex.encodeHexString(digest);
   }
 
   /** Blob data to store in a file and register by server side. */
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 92ec86afc..29bc9cff0 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -34,7 +34,6 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Supplier;
-import javax.xml.bind.DatatypeConverter;
 import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
 import net.snowflake.client.jdbc.internal.snowflake.common.core.SnowflakeDateTimeFormat;
 import net.snowflake.client.jdbc.internal.snowflake.common.util.Power10;
@@ -42,6 +41,8 @@
 import net.snowflake.ingest.streaming.internal.serialization.ZonedDateTimeSerializer;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
 
 /** Utility class for parsing and validating inputs based on Snowflake types */
 class DataValidationUtil {
@@ -592,8 +593,8 @@ static byte[] validateAndParseBinary(
       output = (byte[]) input;
     } else if (input instanceof String) {
       try {
-        output = DatatypeConverter.parseHexBinary((String) input);
-      } catch (IllegalArgumentException e) {
+        output = Hex.decodeHex((String) input);
+      } catch (DecoderException e) {
         throw valueFormatNotAllowedException(columnName, input, "BINARY", "Not a valid hex string");
       }
     } else {
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 76ab1809f..a3a33129c 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -42,9 +42,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import javax.xml.bind.DatatypeConverter;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -747,7 +748,7 @@ public void testValidVariantType() {
   }
 
   @Test
-  public void testValidateAndParseBinary() {
+  public void testValidateAndParseBinary() throws DecoderException {
     byte[] maxAllowedArray = new byte[BYTES_8_MB];
     byte[] maxAllowedArrayMinusOne = new byte[BYTES_8_MB - 1];
 
@@ -759,8 +760,7 @@ public void testValidateAndParseBinary() {
         new byte[] {-1, 0, 1},
         validateAndParseBinary("COL", new byte[] {-1, 0, 1}, Optional.empty()));
     assertArrayEquals(
-        DatatypeConverter.parseHexBinary(
-            "1234567890abcdef"), // pragma: allowlist secret NOT A SECRET
+        Hex.decodeHex("1234567890abcdef"), // pragma: allowlist secret NOT A SECRET
         validateAndParseBinary(
             "COL", "1234567890abcdef", Optional.empty())); // pragma: allowlist secret NOT A SECRET
 

From 39d9924d71bfcf3b74c6dff89cd8bd31c986a060 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Fri, 17 Feb 2023 11:35:05 -0800
Subject: [PATCH 128/356] NO-SNOW: revert
 https://github.com/snowflakedb/snowflake-kafka-connector/pull/546/files as
 it's causing failure on GCS (#334)

revert https://github.com/snowflakedb/snowflake-kafka-connector/pull/546/files as it's causing failure on GCS
---
 pom.xml                                       |  2 +-
 .../snowflake/ingest/SimpleIngestManager.java |  2 +-
 ...nowflakeStreamingIngestClientInternal.java |  5 +-
 .../net/snowflake/ingest/utils/HttpUtil.java  | 49 ++-----------------
 .../internal/StreamingIngestStageTest.java    | 45 -----------------
 5 files changed, 8 insertions(+), 95 deletions(-)

diff --git a/pom.xml b/pom.xml
index 13db98906..71c0df06d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -194,7 +194,7 @@
         <dependency>
             <groupId>net.snowflake</groupId>
             <artifactId>snowflake-jdbc</artifactId>
-            <version>3.13.18</version>
+            <version>3.13.15</version>
         </dependency>
 
         <!-- jwt token for key pair authentication with GS -->
diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
index d167488dd..e5163a4dc 100644
--- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
+++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
@@ -407,7 +407,7 @@ private void init(String account, String user, String pipe, KeyPair keyPair) {
     this.keyPair = keyPair;
 
     // make our client for sending requests
-    httpClient = HttpUtil.getHttpClient(account);
+    httpClient = HttpUtil.getHttpClient();
     // make the request builder we'll use to build messages to the service
   }
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 4ad2c96ec..77e25aa76 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -101,8 +101,6 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
   // Name of the client
   private final String name;
 
-  private String accountName;
-
   // Snowflake role for the client to use
   private String role;
 
@@ -167,9 +165,8 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
     this.parameterProvider = new ParameterProvider(parameterOverrides, prop);
 
     this.name = name;
-    this.accountName = accountURL == null ? null : accountURL.getAccount();
     this.isTestMode = isTestMode;
-    this.httpClient = httpClient == null ? HttpUtil.getHttpClient(accountName) : httpClient;
+    this.httpClient = httpClient == null ? HttpUtil.getHttpClient() : httpClient;
     this.channelCache = new ChannelCache<>();
     this.allocator = new RootAllocator();
     this.isClosed = false;
diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index 72bf32767..3b34459f5 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -14,7 +14,6 @@
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
-import java.util.regex.Pattern;
 import javax.net.ssl.SSLContext;
 import net.snowflake.client.core.SFSessionProperty;
 import net.snowflake.client.jdbc.internal.apache.http.HttpHost;
@@ -48,10 +47,9 @@ public class HttpUtil {
   public static final String USE_PROXY = "http.useProxy";
   public static final String PROXY_HOST = "http.proxyHost";
   public static final String PROXY_PORT = "http.proxyPort";
-  public static final String NON_PROXY_HOSTS = "http.nonProxyHosts";
+
   public static final String HTTP_PROXY_USER = "http.proxyUser";
   public static final String HTTP_PROXY_PASSWORD = "http.proxyPassword";
-  private static final String SNOWFLAKE_DOMAIN_NAME = ".snowflakecomputing.com";
 
   private static final String PROXY_SCHEME = "http";
   private static final String FIRST_FAULT_TIMESTAMP = "FIRST_FAULT_TIMESTAMP";
@@ -86,15 +84,11 @@ public class HttpUtil {
   // Only connections that are currently owned, not checked out, are subject to idle timeouts.
   private static final int DEFAULT_IDLE_CONNECTION_TIMEOUT_SECONDS = 30;
 
-  /**
-   * @param {@code String} account name to connect to (excluding snowflakecomputing.com domain)
-   * @return Instance of CloseableHttpClient
-   */
-  public static CloseableHttpClient getHttpClient(String accountName) {
+  public static CloseableHttpClient getHttpClient() {
     if (httpClient == null) {
       synchronized (HttpUtil.class) {
         if (httpClient == null) {
-          initHttpClient(accountName);
+          initHttpClient();
         }
       }
     }
@@ -106,7 +100,7 @@ public static CloseableHttpClient getHttpClient(String accountName) {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(HttpUtil.class);
 
-  private static void initHttpClient(String accountName) {
+  private static void initHttpClient() {
 
     Security.setProperty("ocsp.enable", "true");
 
@@ -152,7 +146,7 @@ private static void initHttpClient(String accountName) {
             .setDefaultRequestConfig(requestConfig);
 
     // proxy settings
-    if ("true".equalsIgnoreCase(System.getProperty(USE_PROXY)) && !shouldBypassProxy(accountName)) {
+    if ("true".equalsIgnoreCase(System.getProperty(USE_PROXY))) {
       if (System.getProperty(PROXY_PORT) == null) {
         throw new IllegalArgumentException(
             "proxy port number is not provided, please assign proxy port to http.proxyPort option");
@@ -322,12 +316,6 @@ public static Properties generateProxyPropertiesForJDBC() {
         proxyProperties.put(SFSessionProperty.PROXY_USER.getPropertyKey(), proxyUser);
         proxyProperties.put(SFSessionProperty.PROXY_PASSWORD.getPropertyKey(), proxyPassword);
       }
-
-      // Check if http.nonProxyHosts was set
-      final String nonProxyHosts = System.getProperty(NON_PROXY_HOSTS);
-      if (!isNullOrEmpty(nonProxyHosts)) {
-        proxyProperties.put(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey(), nonProxyHosts);
-      }
     }
     return proxyProperties;
   }
@@ -413,31 +401,4 @@ private static String createPoolStatsInfo(String title, PoolStats poolStats) {
     }
     return title;
   }
-
-  /**
-   * Changes the account name to the format accountName.snowflakecomputing.com then returns a
-   * boolean to indicate if we should go through a proxy or not.
-   */
-  public static Boolean shouldBypassProxy(String accountName) {
-    String targetHost = accountName + SNOWFLAKE_DOMAIN_NAME;
-    return System.getProperty(NON_PROXY_HOSTS) != null && isInNonProxyHosts(targetHost);
-  }
-
-  /**
-   * The target hostname input is compared with the hosts in the '|' separated list provided by the
-   * http.nonProxyHosts parameter using regex. The nonProxyHosts will be used as our Patterns, so we
-   * need to replace the '.' and '*' characters since those are special regex constructs that mean
-   * 'any character,' and 'repeat 0 or more times.'
-   */
-  private static Boolean isInNonProxyHosts(String targetHost) {
-    String nonProxyHosts =
-        System.getProperty(NON_PROXY_HOSTS).replace(".", "\\.").replace("*", ".*");
-    String[] nonProxyHostsArray = nonProxyHosts.split("\\|");
-    for (String i : nonProxyHostsArray) {
-      if (Pattern.compile(i).matcher(targetHost).matches()) {
-        return true;
-      }
-    }
-    return false;
-  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
index 45cd82c33..79c809e24 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
@@ -3,12 +3,10 @@
 import static net.snowflake.client.core.Constants.CLOUD_STORAGE_CREDENTIALS_EXPIRED;
 import static net.snowflake.ingest.utils.HttpUtil.HTTP_PROXY_PASSWORD;
 import static net.snowflake.ingest.utils.HttpUtil.HTTP_PROXY_USER;
-import static net.snowflake.ingest.utils.HttpUtil.NON_PROXY_HOSTS;
 import static net.snowflake.ingest.utils.HttpUtil.PROXY_HOST;
 import static net.snowflake.ingest.utils.HttpUtil.PROXY_PORT;
 import static net.snowflake.ingest.utils.HttpUtil.USE_PROXY;
 import static net.snowflake.ingest.utils.HttpUtil.generateProxyPropertiesForJDBC;
-import static net.snowflake.ingest.utils.HttpUtil.shouldBypassProxy;
 import static org.mockito.Mockito.times;
 
 import java.io.ByteArrayInputStream;
@@ -393,13 +391,11 @@ public void testGenerateProxyPropertiesForJDBC() {
     String oldProxyPort = System.getProperty(PROXY_PORT);
     String oldUser = System.getProperty(HTTP_PROXY_USER);
     String oldPassword = System.getProperty(HTTP_PROXY_PASSWORD);
-    String oldNonProxyHosts = System.getProperty(NON_PROXY_HOSTS);
 
     String proxyHost = "localhost";
     String proxyPort = "8080";
     String user = "admin";
     String password = "test";
-    String nonProxyHosts = "*.snowflakecomputing.com";
 
     try {
       // Test empty properties when USE_PROXY is NOT set;
@@ -411,7 +407,6 @@ public void testGenerateProxyPropertiesForJDBC() {
       System.setProperty(PROXY_PORT, proxyPort);
       System.setProperty(HTTP_PROXY_USER, user);
       System.setProperty(HTTP_PROXY_PASSWORD, password);
-      System.setProperty(NON_PROXY_HOSTS, nonProxyHosts);
 
       // Verify that properties are set
       props = generateProxyPropertiesForJDBC();
@@ -420,8 +415,6 @@ public void testGenerateProxyPropertiesForJDBC() {
       Assert.assertEquals(proxyPort, props.get(SFSessionProperty.PROXY_PORT.getPropertyKey()));
       Assert.assertEquals(user, props.get(SFSessionProperty.PROXY_USER.getPropertyKey()));
       Assert.assertEquals(password, props.get(SFSessionProperty.PROXY_PASSWORD.getPropertyKey()));
-      Assert.assertEquals(
-          nonProxyHosts, props.get(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey()));
     } finally {
       // Cleanup
       if (oldUseProxy != null) {
@@ -433,44 +426,6 @@ public void testGenerateProxyPropertiesForJDBC() {
         System.setProperty(HTTP_PROXY_USER, oldUser);
         System.setProperty(HTTP_PROXY_PASSWORD, oldPassword);
       }
-      if (oldNonProxyHosts != null) {
-        System.setProperty(NON_PROXY_HOSTS, oldNonProxyHosts);
-      }
-    }
-  }
-
-  @Test
-  public void testShouldBypassProxy() {
-    String oldNonProxyHosts = System.getProperty(NON_PROXY_HOSTS);
-    String accountName = "accountName12345";
-    String accountDashName = "account-name12345";
-    String accountUnderscoreName = "account_name12345";
-    String nonProxyHosts = "*.snowflakecomputing.com|localhost";
-
-    System.setProperty(NON_PROXY_HOSTS, nonProxyHosts);
-    Assert.assertTrue(shouldBypassProxy(accountName));
-    Assert.assertTrue(shouldBypassProxy(accountDashName));
-    Assert.assertTrue(shouldBypassProxy(accountUnderscoreName));
-
-    String accountNamePrivateLink = "accountName12345.privatelink";
-    nonProxyHosts = "*.privatelink.snowflakecomputing.com|localhost";
-    System.setProperty(NON_PROXY_HOSTS, nonProxyHosts);
-    Assert.assertTrue(shouldBypassProxy(accountNamePrivateLink));
-
-    // Previous tests should return false with the new nonProxyHosts value
-    Assert.assertFalse(shouldBypassProxy(accountName));
-    Assert.assertFalse(shouldBypassProxy(accountDashName));
-    Assert.assertFalse(shouldBypassProxy(accountUnderscoreName));
-
-    // All tests should return false after clearing the nonProxyHosts property
-    System.clearProperty(NON_PROXY_HOSTS);
-    Assert.assertFalse(shouldBypassProxy(accountName));
-    Assert.assertFalse(shouldBypassProxy(accountDashName));
-    Assert.assertFalse(shouldBypassProxy(accountUnderscoreName));
-    Assert.assertFalse(shouldBypassProxy(accountNamePrivateLink));
-
-    if (oldNonProxyHosts != null) {
-      System.setProperty(NON_PROXY_HOSTS, oldNonProxyHosts);
     }
   }
 }

From b3ae77576a031d58db460c5faf4a9c2f151b2d6e Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Fri, 17 Feb 2023 13:52:50 -0800
Subject: [PATCH 129/356] V1.0.3-beta.1 release (#335)

The 1.0.3-beta release has an issue with GCS, so we have to revert #334 and then create a new release on top
---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 71c0df06d..b3b18e11c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
     <!-- Arifact name and version information -->
     <groupId>net.snowflake</groupId>
     <artifactId>snowflake-ingest-sdk</artifactId>
-    <version>1.0.3-beta</version>
+    <version>1.0.3-beta.1</version>
     <packaging>jar</packaging>
     <name>Snowflake Ingest SDK</name>
     <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 16f0b8d9d..880793fbb 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -127,7 +127,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.0.3-beta";
+  public static final String DEFAULT_VERSION = "1.0.3-beta.1";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 15e5ae4b60d8fd719ae233441633877031c06e76 Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Mon, 20 Feb 2023 19:31:31 +0100
Subject: [PATCH 130/356] SNOW-738614 improve uncompressed size estimation for
 Parquet row buffers (#333)

---
 .../internal/ParquetValueParser.java          | 56 ++++++++++--
 .../internal/ParquetValueParserTest.java      | 86 +++++++++++++++----
 2 files changed, 115 insertions(+), 27 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
index 3ac5b80ca..8d1f131fc 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
@@ -17,6 +17,38 @@
 
 /** Parses a user column value into Parquet internal representation for buffering. */
 class ParquetValueParser {
+
+  // Parquet uses BitPacking to encode boolean, hence 1 bit per value
+  public static final float BIT_ENCODING_BYTE_LEN = 1.0f / 8;
+
+  /**
+   * On average parquet needs 2 bytes / 8 values for the RLE+bitpack encoded definition level.
+   *
+   * <ul>
+   *   There are two cases how definition level (0 for null values, 1 for non-null values) is
+   *   encoded:
+   *   <li>If there are at least 8 repeated values in a row, they are run-length encoded (length +
+   *       value itself). E.g. 11111111 -> 8 1
+   *   <li>If there are less than 8 repeated values, they are written in group as part of a
+   *       bit-length encoded run, e.g. 1111 -> 15 A bit-length encoded run ends when either 64
+   *       groups of 8 values have been written or if a new RLE run starts.
+   *       <p>To distinguish between RLE and bitpack run, there is 1 extra bytes written as header
+   *       when a bitpack run starts.
+   * </ul>
+   *
+   * <ul>
+   *   For more details see ColumnWriterV1#createDLWriter and {@link
+   *   org.apache.parquet.column.values.rle.RunLengthBitPackingHybridEncoder#writeInt(int)}
+   * </ul>
+   *
+   * <p>Since we don't have nested types, repetition level is always 0 and is not stored at all by
+   * Parquet.
+   */
+  public static final float DEFINITION_LEVEL_ENCODING_BYTE_LEN = 2.0f / 8;
+
+  // Parquet stores length in 4 bytes before the actual data bytes
+  public static final int BYTE_ARRAY_LENGTH_ENCODING_BYTE_LEN = 4;
+
   /** Parquet internal value representation for buffering. */
   static class ParquetBufferValue {
     private final Object value;
@@ -52,7 +84,8 @@ static ParquetBufferValue parseColumnValueToParquet(
       RowBufferStats stats,
       ZoneId defaultTimezone) {
     Utils.assertNotNull("Parquet column stats", stats);
-    float size = 0F;
+    float estimatedParquetSize = 0F;
+    estimatedParquetSize += DEFINITION_LEVEL_ENCODING_BYTE_LEN;
     if (value != null) {
       AbstractRowBuffer.ColumnLogicalType logicalType =
           AbstractRowBuffer.ColumnLogicalType.valueOf(columnMetadata.getLogicalType());
@@ -64,7 +97,7 @@ static ParquetBufferValue parseColumnValueToParquet(
               DataValidationUtil.validateAndParseBoolean(columnMetadata.getName(), value);
           value = intValue > 0;
           stats.addIntValue(BigInteger.valueOf(intValue));
-          size = 1;
+          estimatedParquetSize += BIT_ENCODING_BYTE_LEN;
           break;
         case INT32:
           int intVal =
@@ -77,7 +110,7 @@ static ParquetBufferValue parseColumnValueToParquet(
                   physicalType);
           value = intVal;
           stats.addIntValue(BigInteger.valueOf(intVal));
-          size = 4;
+          estimatedParquetSize += 4;
           break;
         case INT64:
           long longValue =
@@ -91,26 +124,31 @@ static ParquetBufferValue parseColumnValueToParquet(
                   defaultTimezone);
           value = longValue;
           stats.addIntValue(BigInteger.valueOf(longValue));
-          size = 8;
+          estimatedParquetSize += 8;
           break;
         case DOUBLE:
           double doubleValue =
               DataValidationUtil.validateAndParseReal(columnMetadata.getName(), value);
           value = doubleValue;
           stats.addRealValue(doubleValue);
-          size = 8;
+          estimatedParquetSize += 8;
           break;
         case BINARY:
+          int length = 0;
           if (logicalType == AbstractRowBuffer.ColumnLogicalType.BINARY) {
             value = getBinaryValueForLogicalBinary(value, stats, columnMetadata);
-            size = ((byte[]) value).length;
+            length = ((byte[]) value).length;
           } else {
             String str = getBinaryValue(value, stats, columnMetadata);
             value = str;
             if (str != null) {
-              size = str.getBytes().length;
+              length = str.getBytes().length;
             }
           }
+          if (value != null) {
+            estimatedParquetSize += (BYTE_ARRAY_LENGTH_ENCODING_BYTE_LEN + length);
+          }
+
           break;
         case FIXED_LEN_BYTE_ARRAY:
           BigInteger intRep =
@@ -124,7 +162,7 @@ static ParquetBufferValue parseColumnValueToParquet(
                   defaultTimezone);
           stats.addIntValue(intRep);
           value = getSb16Bytes(intRep);
-          size += 16;
+          estimatedParquetSize += 16;
           break;
         default:
           throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
@@ -139,7 +177,7 @@ static ParquetBufferValue parseColumnValueToParquet(
       stats.incCurrentNullCount();
     }
 
-    return new ParquetBufferValue(value, size);
+    return new ParquetBufferValue(value, estimatedParquetSize);
   }
 
   /**
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
index 15be2cdb2..71237f5f9 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
@@ -1,6 +1,9 @@
 package net.snowflake.ingest.streaming.internal;
 
 import static java.time.ZoneOffset.UTC;
+import static net.snowflake.ingest.streaming.internal.ParquetValueParser.BIT_ENCODING_BYTE_LEN;
+import static net.snowflake.ingest.streaming.internal.ParquetValueParser.BYTE_ARRAY_LENGTH_ENCODING_BYTE_LEN;
+import static net.snowflake.ingest.streaming.internal.ParquetValueParser.DEFINITION_LEVEL_ENCODING_BYTE_LEN;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -35,7 +38,7 @@ public void parseValueFixedSB1ToInt32() {
         .rowBufferStats(rowBufferStats)
         .expectedValueClass(Integer.class)
         .expectedParsedValue(12)
-        .expectedSize(4.0f)
+        .expectedSize(4.0f + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
         .expectedMinMax(BigInteger.valueOf(12))
         .assertMatches();
   }
@@ -61,7 +64,7 @@ public void parseValueFixedSB2ToInt32() {
         .rowBufferStats(rowBufferStats)
         .expectedValueClass(Integer.class)
         .expectedParsedValue(1234)
-        .expectedSize(4.0f)
+        .expectedSize(4.0f + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
         .expectedMinMax(BigInteger.valueOf(1234))
         .assertMatches();
   }
@@ -87,7 +90,7 @@ public void parseValueFixedSB4ToInt32() {
         .rowBufferStats(rowBufferStats)
         .expectedValueClass(Integer.class)
         .expectedParsedValue(123456789)
-        .expectedSize(4.0f)
+        .expectedSize(4.0f + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
         .expectedMinMax(BigInteger.valueOf(123456789))
         .assertMatches();
   }
@@ -117,7 +120,7 @@ public void parseValueFixedSB8ToInt64() {
         .rowBufferStats(rowBufferStats)
         .expectedValueClass(Long.class)
         .expectedParsedValue(123456789987654321L)
-        .expectedSize(8.0f)
+        .expectedSize(8.0f + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
         .expectedMinMax(BigInteger.valueOf(123456789987654321L))
         .assertMatches();
   }
@@ -149,7 +152,7 @@ public void parseValueFixedSB16ToByteArray() {
         .expectedParsedValue(
             ParquetValueParser.getSb16Bytes(
                 new BigInteger("91234567899876543219876543211234567891")))
-        .expectedSize(16.0f)
+        .expectedSize(16.0f + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
         .expectedMinMax(new BigInteger("91234567899876543219876543211234567891"))
         .assertMatches();
   }
@@ -179,7 +182,7 @@ public void parseValueFixedDecimalToInt32() {
         .rowBufferStats(rowBufferStats)
         .expectedValueClass(Double.class)
         .expectedParsedValue(Double.valueOf("12345.54321"))
-        .expectedSize(8.0f)
+        .expectedSize(8.0f + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
         .expectedMinMax(Double.valueOf("12345.54321"))
         .assertMatches();
   }
@@ -203,7 +206,7 @@ public void parseValueDouble() {
         .rowBufferStats(rowBufferStats)
         .expectedValueClass(Double.class)
         .expectedParsedValue(Double.valueOf(12345.54321))
-        .expectedSize(8.0f)
+        .expectedSize(8.0f + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
         .expectedMinMax(Double.valueOf(12345.54321))
         .assertMatches();
   }
@@ -227,7 +230,7 @@ public void parseValueBoolean() {
         .rowBufferStats(rowBufferStats)
         .expectedValueClass(Boolean.class)
         .expectedParsedValue(true)
-        .expectedSize(1.0f)
+        .expectedSize(BIT_ENCODING_BYTE_LEN + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
         .expectedMinMax(BigInteger.valueOf(1))
         .assertMatches();
   }
@@ -255,7 +258,8 @@ public void parseValueBinary() {
         .rowBufferStats(rowBufferStats)
         .expectedValueClass(byte[].class)
         .expectedParsedValue("1234abcd".getBytes())
-        .expectedSize(8.0f)
+        .expectedSize(
+            BYTE_ARRAY_LENGTH_ENCODING_BYTE_LEN + 8.0f + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
         .expectedMinMax("1234abcd".getBytes(StandardCharsets.UTF_8))
         .assertMatches();
   }
@@ -291,7 +295,10 @@ private void testJsonWithLogicalType(String logicalType) {
         .rowBufferStats(rowBufferStats)
         .expectedValueClass(String.class)
         .expectedParsedValue(var)
-        .expectedSize(var.getBytes().length)
+        .expectedSize(
+            BYTE_ARRAY_LENGTH_ENCODING_BYTE_LEN
+                + var.getBytes().length
+                + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
         .expectedMinMax(null)
         .assertMatches();
   }
@@ -361,11 +368,46 @@ public void parseValueArrayToBinary() {
         .rowBufferStats(rowBufferStats)
         .expectedValueClass(String.class)
         .expectedParsedValue(resultArray)
-        .expectedSize(resultArray.length())
+        .expectedSize(
+            BYTE_ARRAY_LENGTH_ENCODING_BYTE_LEN
+                + resultArray.length()
+                + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
         .expectedMinMax(null)
         .assertMatches();
   }
 
+  @Test
+  public void parseValueTextToBinary() {
+    ColumnMetadata testCol =
+        ColumnMetadataBuilder.newBuilder()
+            .logicalType("TEXT")
+            .physicalType("LOB")
+            .nullable(true)
+            .length(56)
+            .build();
+
+    String text = "This is a sample text! Length is bigger than 32 bytes :)";
+
+    RowBufferStats rowBufferStats = new RowBufferStats("COL1");
+    ParquetValueParser.ParquetBufferValue pv =
+        ParquetValueParser.parseColumnValueToParquet(
+            text, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats, UTC);
+
+    String result = text;
+
+    ParquetValueParserAssertionBuilder.newBuilder()
+        .parquetBufferValue(pv)
+        .rowBufferStats(rowBufferStats)
+        .expectedValueClass(String.class)
+        .expectedParsedValue(result)
+        .expectedSize(
+            BYTE_ARRAY_LENGTH_ENCODING_BYTE_LEN
+                + result.length()
+                + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
+        .expectedMinMax(text) // min/max are truncated later to 32 bytes, not in the parsing step.
+        .assertMatches();
+  }
+
   @Test
   public void parseValueTimestampNtzSB4Error() {
     ColumnMetadata testCol =
@@ -417,7 +459,7 @@ public void parseValueTimestampNtzSB8ToINT64() {
         .rowBufferStats(rowBufferStats)
         .expectedValueClass(Long.class)
         .expectedParsedValue(1367182621000L)
-        .expectedSize(8.0f)
+        .expectedSize(8.0f + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
         .expectedMinMax(BigInteger.valueOf(1367182621000L))
         .assertMatches();
   }
@@ -447,7 +489,7 @@ public void parseValueTimestampNtzSB16ToByteArray() {
         .expectedValueClass(byte[].class)
         .expectedParsedValue(
             ParquetValueParser.getSb16Bytes(BigInteger.valueOf(1663538707123456789L)))
-        .expectedSize(16.0f)
+        .expectedSize(16.0f + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
         .expectedMinMax(BigInteger.valueOf(1663538707123456789L))
         .assertMatches();
   }
@@ -472,7 +514,7 @@ public void parseValueDateToInt32() {
         .rowBufferStats(rowBufferStats)
         .expectedValueClass(Integer.class)
         .expectedParsedValue(Integer.valueOf(18628))
-        .expectedSize(4.0f)
+        .expectedSize(4.0f + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
         .expectedMinMax(BigInteger.valueOf(18628))
         .assertMatches();
   }
@@ -497,7 +539,7 @@ public void parseValueTimeSB4ToInt32() {
         .rowBufferStats(rowBufferStats)
         .expectedValueClass(Integer.class)
         .expectedParsedValue(3600)
-        .expectedSize(4.0f)
+        .expectedSize(4.0f + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
         .expectedMinMax(BigInteger.valueOf(3600))
         .assertMatches();
   }
@@ -522,7 +564,7 @@ public void parseValueTimeSB8ToInt64() {
         .rowBufferStats(rowBufferStats)
         .expectedValueClass(Long.class)
         .expectedParsedValue(3600123L)
-        .expectedSize(8.0f)
+        .expectedSize(8.0f + DEFINITION_LEVEL_ENCODING_BYTE_LEN)
         .expectedMinMax(BigInteger.valueOf(3600123))
         .assertMatches();
   }
@@ -621,8 +663,16 @@ void assertMatches() {
         return;
       } else if (valueClass.equals(String.class)) {
         // String can have null min/max stats for variant data types
-        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinStrValue());
-        Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMaxStrValue());
+        Object min =
+            rowBufferStats.getCurrentMinStrValue() != null
+                ? new String(rowBufferStats.getCurrentMinStrValue(), StandardCharsets.UTF_8)
+                : rowBufferStats.getCurrentMinStrValue();
+        Object max =
+            rowBufferStats.getCurrentMaxStrValue() != null
+                ? new String(rowBufferStats.getCurrentMaxStrValue(), StandardCharsets.UTF_8)
+                : rowBufferStats.getCurrentMaxStrValue();
+        Assert.assertEquals(minMaxStat, min);
+        Assert.assertEquals(minMaxStat, max);
         return;
       } else if (minMaxStat instanceof Double || minMaxStat instanceof BigDecimal) {
         Assert.assertEquals(minMaxStat, rowBufferStats.getCurrentMinRealValue());

From 9ee5792ef4f0b60213a2f479ba98d0198e76151c Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Wed, 22 Feb 2023 20:11:13 +0100
Subject: [PATCH 131/356] SNOW-745946 SNOW-735517 Create defensive copy of
 ingested rows (#336)

---
 .../streaming/internal/DataValidationUtil.java    | 15 ++++++---------
 .../SnowflakeStreamingIngestChannelInternal.java  | 13 ++++++++++++-
 2 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 29bc9cff0..9fb993d31 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -90,14 +90,6 @@ class DataValidationUtil {
     objectMapper.registerModule(module);
   }
 
-  /**
-   * Creates a new SnowflakeDateTimeFormat. In order to avoid SnowflakeDateTimeFormat's
-   * synchronization blocks, we create a new instance when needed instead of sharing one instance.
-   */
-  private static SnowflakeDateTimeFormat createDateTimeFormatter() {
-    return SnowflakeDateTimeFormat.fromSqlFormat("auto");
-  }
-
   /**
    * Validates and parses input as JSON. All types in the object tree must be valid variant types,
    * see {@link DataValidationUtil#isAllowedSemiStructuredType}.
@@ -590,7 +582,12 @@ static byte[] validateAndParseBinary(
       String columnName, Object input, Optional<Integer> maxLengthOptional) {
     byte[] output;
     if (input instanceof byte[]) {
-      output = (byte[]) input;
+      // byte[] is a mutable object, we need to create a defensive copy to protect against
+      // concurrent modifications of the array, which could lead to mismatch between data
+      // and metadata
+      byte[] originalInputArray = (byte[]) input;
+      output = new byte[originalInputArray.length];
+      System.arraycopy(originalInputArray, 0, output, 0, originalInputArray.length);
     } else if (input instanceof String) {
       try {
         output = Hex.decodeHex((String) input);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 982f3290d..df17bb6a8 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -14,6 +14,8 @@
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
@@ -353,7 +355,16 @@ public InsertValidationResponse insertRows(
       throw new SFException(ErrorCode.CLOSED_CHANNEL, getFullyQualifiedName());
     }
 
-    InsertValidationResponse response = this.rowBuffer.insertRows(rows, offsetToken);
+    // We create a shallow copy to protect against concurrent addition/removal of columns, which can
+    // lead to double counting of null values, for example. Individual mutable values may still be
+    // concurrently modified (e.g. byte[]). Before validation and EP calculation, we must make sure
+    // that defensive copies of all mutable objects are created.
+    final List<Map<String, Object>> rowsCopy = new LinkedList<>();
+    for (Map<String, Object> row : rows) {
+      rowsCopy.add(new HashMap<>(row));
+    }
+
+    InsertValidationResponse response = this.rowBuffer.insertRows(rowsCopy, offsetToken);
 
     // Start flush task if the chunk size reaches a certain size
     // TODO: Checking table/chunk level size reduces throughput a lot, we may want to check it only

From 530980d1ed66edd1de2580e72ca06178ce7daba6 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Fri, 24 Feb 2023 17:53:20 +0100
Subject: [PATCH 132/356] SNOW-749216 Allow leading and trailing whitespace
 (#339)

In order ot be compatible with Snowflake, we should allow surrounding
whitespace around string input values.
---
 .../internal/DataValidationUtil.java          | 28 ++++++-------
 ...owflakeStreamingIngestChannelInternal.java |  4 +-
 .../internal/DataValidationUtilTest.java      | 39 ++++++++++++++++++-
 .../internal/datatypes/StringsIT.java         |  1 +
 4 files changed, 54 insertions(+), 18 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 9fb993d31..73adc2e4e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -343,7 +343,7 @@ private static OffsetDateTime inputToOffsetDateTime(
     }
 
     if (input instanceof String) {
-      String stringInput = (String) input;
+      String stringInput = ((String) input).trim();
       {
         // First, try to parse ZonedDateTime
         ZonedDateTime zoned = catchParsingError(() -> ZonedDateTime.parse(stringInput));
@@ -532,7 +532,8 @@ static BigDecimal validateAndParseBigDecimal(String columnName, Object input) {
       return BigDecimal.valueOf(((Number) input).doubleValue());
     } else if (input instanceof String) {
       try {
-        return new BigDecimal((String) input);
+        final String stringInput = ((String) input).trim();
+        return new BigDecimal(stringInput);
       } catch (NumberFormatException e) {
         throw valueFormatNotAllowedException(columnName, input, "NUMBER", "Not a valid number");
       }
@@ -590,7 +591,8 @@ static byte[] validateAndParseBinary(
       System.arraycopy(originalInputArray, 0, output, 0, originalInputArray.length);
     } else if (input instanceof String) {
       try {
-        output = Hex.decodeHex((String) input);
+        String stringInput = ((String) input).trim();
+        output = Hex.decodeHex(stringInput);
       } catch (DecoderException e) {
         throw valueFormatNotAllowedException(columnName, input, "BINARY", "Not a valid hex string");
       }
@@ -627,7 +629,7 @@ static BigInteger validateAndParseTime(String columnName, Object input, int scal
     } else if (input instanceof OffsetTime) {
       return validateAndParseTime(columnName, ((OffsetTime) input).toLocalTime(), scale);
     } else if (input instanceof String) {
-      String stringInput = (String) input;
+      String stringInput = ((String) input).trim();
       {
         // First, try to parse LocalTime
         LocalTime localTime = catchParsingError(() -> LocalTime.parse(stringInput));
@@ -712,7 +714,7 @@ static double validateAndParseReal(String columnName, Object input) {
     } else if (input instanceof Number) {
       return ((Number) input).doubleValue();
     } else if (input instanceof String) {
-      String stringInput = (String) input;
+      String stringInput = ((String) input).trim();
       try {
         return Double.parseDouble(stringInput);
       } catch (NumberFormatException err) {
@@ -776,8 +778,8 @@ static void checkValueInRange(BigDecimal bigDecimalValue, int scale, int precisi
       Sets.newHashSet("1", "0", "yes", "no", "y", "n", "t", "f", "true", "false", "on", "off");
 
   private static boolean convertStringToBoolean(String columnName, String value) {
-    String lowerCasedValue = value.toLowerCase();
-    if (!allowedBooleanStringsLowerCased.contains(lowerCasedValue)) {
+    String normalizedInput = value.toLowerCase().trim();
+    if (!allowedBooleanStringsLowerCased.contains(normalizedInput)) {
       throw valueFormatNotAllowedException(
           columnName,
           value,
@@ -786,12 +788,12 @@ private static boolean convertStringToBoolean(String columnName, String value) {
               + " https://docs.snowflake.com/en/sql-reference/data-types-logical.html#conversion-to-boolean"
               + " for the list of supported formats");
     }
-    return "1".equals(lowerCasedValue)
-        || "yes".equals(lowerCasedValue)
-        || "y".equals(lowerCasedValue)
-        || "t".equals(lowerCasedValue)
-        || "true".equals(lowerCasedValue)
-        || "on".equals(lowerCasedValue);
+    return "1".equals(normalizedInput)
+        || "yes".equals(normalizedInput)
+        || "y".equals(normalizedInput)
+        || "t".equals(normalizedInput)
+        || "true".equals(normalizedInput)
+        || "on".equals(normalizedInput);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index df17bb6a8..12cf0a15a 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -360,9 +360,7 @@ public InsertValidationResponse insertRows(
     // concurrently modified (e.g. byte[]). Before validation and EP calculation, we must make sure
     // that defensive copies of all mutable objects are created.
     final List<Map<String, Object>> rowsCopy = new LinkedList<>();
-    for (Map<String, Object> row : rows) {
-      rowsCopy.add(new HashMap<>(row));
-    }
+    rows.forEach(r -> rowsCopy.add(new HashMap<>(r)));
 
     InsertValidationResponse response = this.rowBuffer.insertRows(rowsCopy, offsetToken);
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index a3a33129c..58948bd17 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -94,6 +94,7 @@ public void testValidateAndParseDate() {
     assertEquals(19380, validateAndParseDate("COL", Instant.ofEpochMilli(1674478926000L)));
 
     assertEquals(-923, validateAndParseDate("COL", "1967-06-23"));
+    assertEquals(-923, validateAndParseDate("COL", "  1967-06-23 \t\n"));
     assertEquals(-923, validateAndParseDate("COL", "1967-06-23T01:01:01"));
     assertEquals(18464, validateAndParseDate("COL", "2020-07-21"));
     assertEquals(18464, validateAndParseDate("COL", "2020-07-21T23:31:00"));
@@ -133,6 +134,7 @@ public void testValidateAndParseDate() {
   public void testValidateAndParseTime() {
     // Test local time
     assertEquals(46920, validateAndParseTime("COL", "13:02", 0).longValueExact());
+    assertEquals(46920, validateAndParseTime("COL", "  13:02 \t\n", 0).longValueExact());
     assertEquals(46926, validateAndParseTime("COL", "13:02:06", 0).longValueExact());
     assertEquals(469260, validateAndParseTime("COL", "13:02:06", 1).longValueExact());
     assertEquals(46926000000000L, validateAndParseTime("COL", "13:02:06", 9).longValueExact());
@@ -225,7 +227,7 @@ public void testValidateAndParseTimestamp() {
     assertEquals(3600, wrapper.getTimezoneOffsetSeconds());
     assertEquals(1500, wrapper.getTimeZoneIndex());
 
-    wrapper = validateAndParseTimestamp("COL", "2021-01-01T01:00:00.123", 9, UTC, true);
+    wrapper = validateAndParseTimestamp("COL", "  2021-01-01T01:00:00.123 \t\n", 9, UTC, true);
     Assert.assertEquals(1609462800, wrapper.getEpoch());
     Assert.assertEquals(123000000, wrapper.getFraction());
     Assert.assertEquals(new BigInteger("1609462800123000000"), wrapper.toBinary(false));
@@ -279,6 +281,13 @@ public void testValidateAndParseTimestamp() {
   @Test
   public void testValidateAndParseBigDecimal() {
     assertEquals(new BigDecimal("1"), validateAndParseBigDecimal("COL", "1"));
+    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal("COL", "  1 \t\n "));
+    assertEquals(
+        new BigDecimal("1000").toBigInteger(),
+        validateAndParseBigDecimal("COL", "1e3").toBigInteger());
+    assertEquals(
+        new BigDecimal("1000").toBigInteger(),
+        validateAndParseBigDecimal("COL", "  1e3 \t\n").toBigInteger());
     assertEquals(
         new BigDecimal("1000").toBigInteger(),
         validateAndParseBigDecimal("COL", "1e3").toBigInteger());
@@ -381,6 +390,7 @@ public void testValidateAndParseVariant() throws Exception {
     assertEquals("1", validateAndParseVariant("COL", "                          1   "));
     String stringVariant = "{\"key\":1}";
     assertEquals(stringVariant, validateAndParseVariant("COL", stringVariant));
+    assertEquals(stringVariant, validateAndParseVariant("COL", "  " + stringVariant + " \t\n"));
 
     // Test custom serializers
     assertEquals(
@@ -439,6 +449,8 @@ public void testValidateAndParseArray() throws Exception {
     assertEquals("[1]", validateAndParseArray("COL", 1));
     assertEquals("[1]", validateAndParseArray("COL", "1"));
     assertEquals("[1]", validateAndParseArray("COL", "                          1   "));
+    assertEquals("[1,2,3]", validateAndParseArray("COL", "[1, 2, 3]"));
+    assertEquals("[1,2,3]", validateAndParseArray("COL", "  [1, 2, 3] \t\n"));
     int[] intArray = new int[] {1, 2, 3};
     assertEquals("[1,2,3]", validateAndParseArray("COL", intArray));
 
@@ -497,6 +509,7 @@ public void testValidateAndParseArray() throws Exception {
   public void testValidateAndParseObject() throws Exception {
     String stringObject = "{\"key\":1}";
     assertEquals(stringObject, validateAndParseObject("COL", stringObject));
+    assertEquals(stringObject, validateAndParseObject("COL", "  " + stringObject + " \t\n"));
 
     String badObject = "foo";
     try {
@@ -763,6 +776,12 @@ public void testValidateAndParseBinary() throws DecoderException {
         Hex.decodeHex("1234567890abcdef"), // pragma: allowlist secret NOT A SECRET
         validateAndParseBinary(
             "COL", "1234567890abcdef", Optional.empty())); // pragma: allowlist secret NOT A SECRET
+    assertArrayEquals(
+        Hex.decodeHex("1234567890abcdef"), // pragma: allowlist secret NOT A SECRET
+        validateAndParseBinary(
+            "COL",
+            "  1234567890abcdef \t\n",
+            Optional.empty())); // pragma: allowlist secret NOT A SECRET
 
     assertArrayEquals(
         maxAllowedArray, validateAndParseBinary("COL", maxAllowedArray, Optional.empty()));
@@ -813,8 +832,10 @@ public void testValidateAndParseReal() throws Exception {
     assertEquals(Double.NaN, validateAndParseReal("COL", "Nan"), 0);
     assertEquals(Double.POSITIVE_INFINITY, validateAndParseReal("COL", "inF"), 0);
     assertEquals(Double.NEGATIVE_INFINITY, validateAndParseReal("COL", "-inF"), 0);
+    assertEquals(Double.NEGATIVE_INFINITY, validateAndParseReal("COL", " -inF \t\n"), 0);
 
     // From string
+    assertEquals(1.23d, validateAndParseReal("COL", "   1.23 \t\n"), 0);
     assertEquals(1.23d, validateAndParseReal("COL", "1.23"), 0);
     assertEquals(123d, validateAndParseReal("COL", "1.23E2"), 0);
     assertEquals(123d, validateAndParseReal("COL", "1.23e2"), 0);
@@ -832,7 +853,21 @@ public void testValidateAndParseBoolean() {
 
     for (Object input :
         Arrays.asList(
-            true, "true", "True", "TruE", "t", "yes", "YeS", "y", "on", "1", 1.1, -1.1, -10, 10)) {
+            true,
+            "true",
+            "True",
+            "TruE",
+            "t",
+            "yes",
+            "YeS",
+            "y",
+            "on",
+            "1",
+            "  true \t\n",
+            1.1,
+            -1.1,
+            -10,
+            10)) {
       assertEquals(1, validateAndParseBoolean("COL", input));
     }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
index 2af1f5803..f41e24dc1 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
@@ -25,6 +25,7 @@ public StringsIT(String name, Constants.BdecVersion bdecVersion) {
   public void testStrings() throws Exception {
     testJdbcTypeCompatibility("VARCHAR", "", new StringProvider());
     testJdbcTypeCompatibility("VARCHAR", "foo", new StringProvider());
+    testJdbcTypeCompatibility("VARCHAR", "  foo  \t\n", new StringProvider());
 
     // Test strings with limited size
     testJdbcTypeCompatibility("VARCHAR(1)", "", new StringProvider());

From fb6ae56cc7683263f086b6b78f61af0414589f2b Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Fri, 24 Feb 2023 12:38:09 -0800
Subject: [PATCH 133/356] V1.1.0 release (#338)

This will be the first non-beta release after v0.10.8 and it contains the support to do Snowpipe Streaming
---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index b3b18e11c..7048aae29 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
     <!-- Arifact name and version information -->
     <groupId>net.snowflake</groupId>
     <artifactId>snowflake-ingest-sdk</artifactId>
-    <version>1.0.3-beta.1</version>
+    <version>1.1.0</version>
     <packaging>jar</packaging>
     <name>Snowflake Ingest SDK</name>
     <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 880793fbb..38c8a0e0d 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -127,7 +127,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.0.3-beta.1";
+  public static final String DEFAULT_VERSION = "1.1.0";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 176b5dc3c0a6af6a599ad083f7b2151791c21fa8 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Tue, 28 Feb 2023 15:55:18 -0800
Subject: [PATCH 134/356] NO-SNOW: add logging to debug file uploading failure
 (#340)

Customers are seeing errors from JDBC during file upload, add some logging to help us debug

KafkaEventaConsumer_7.log:2023-02-25 09:58:42.188 ERROR 1530944 --- [pload-thread-36] n.s.c.jdbc.SnowflakeFileTransferAgent : Exception encountered during file upload:
KafkaEventaConsumer_7.log:2023-02-25 09:58:43.306 ERROR 1530944 --- [upload-thread-3] n.s.c.jdbc.SnowflakeFileTransferAgent : Exception encountered during file upload:
KafkaEventaConsumer_7.log:2023-02-25 09:58:45.964 ERROR 1530944 --- [load-thread-104] n.s.c.jdbc.SnowflakeFileTransferAgent : Exception encountered during file upload:
KafkaEventaConsumer_7.log:2023-02-25 09:58:46.609 ERROR 1530944 --- [pload-thread-12] n.s.c.jdbc.SnowflakeFileTransferAgent : Exception encountered during file upload:
KafkaEventaConsumer_7.log:2023-02-25 09:58:46.644 ERROR 1530944 --- [pload-thread-23] n.s.c.jdbc.SnowflakeFileTransferAgent : Exception encountered during file upload:
KafkaEventaConsumer_7.log:2023-02-25 09:58:47.494 ERROR 1530944 --- [pload-thread-74] n.s.c.jdbc.SnowflakeFileTransferAgent : Exception encountered during file upload:
---
 .../ingest/streaming/internal/StreamingIngestStage.java     | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index c389f3d3e..d3b285cd5 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -36,6 +36,7 @@
 import net.snowflake.ingest.connection.IngestResponseException;
 import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.Utils;
 import org.apache.arrow.util.VisibleForTesting;
@@ -47,6 +48,8 @@ class StreamingIngestStage {
       TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES);
   static final int MAX_RETRY_COUNT = 1;
 
+  private static final Logging logger = new Logging(StreamingIngestStage.class);
+
   /**
    * Wrapper class containing SnowflakeFileTransferMetadata and the timestamp at which the metadata
    * was refreshed
@@ -186,6 +189,8 @@ private void putRemote(String fullFilePath, byte[] data, int retryCount)
               .build());
     } catch (SnowflakeSQLException e) {
       if (e.getErrorCode() != CLOUD_STORAGE_CREDENTIALS_EXPIRED || retryCount >= MAX_RETRY_COUNT) {
+        logger.logError(
+            "Failed to upload to stage, client={}, message={}", clientName, e.getMessage());
         throw e;
       }
       this.refreshSnowflakeMetadata();
@@ -197,6 +202,7 @@ private void putRemote(String fullFilePath, byte[] data, int retryCount)
 
   SnowflakeFileTransferMetadataWithAge refreshSnowflakeMetadata()
       throws SnowflakeSQLException, IOException {
+    logger.logInfo("Refresh Snowflake metadata, client={}", clientName);
     return refreshSnowflakeMetadata(false);
   }
 

From 6f87823c35fc830073fa5bff5563dd4e15ae4c71 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Fri, 3 Mar 2023 19:22:04 +0100
Subject: [PATCH 135/356] Clarify thread-safety guarantees in Javadoc (#342)

---
 README.md                                      |  2 +-
 .../SnowflakeStreamingIngestChannel.java       | 17 +++++++++++------
 .../SnowflakeStreamingIngestClient.java        |  2 ++
 .../streaming/internal/DataValidationUtil.java |  6 ++----
 .../internal/DataValidationUtilTest.java       | 18 +++++++++---------
 5 files changed, 25 insertions(+), 20 deletions(-)

diff --git a/README.md b/README.md
index 843bac370..8d5b455a0 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@ The Snowflake Ingest Service SDK allows users to ingest files into their
 Snowflake data warehouse in a programmatic fashion via key-pair
 authentication. Currently, we support ingestion through the following APIs:
 1. [Snowpipe](https://docs.snowflake.com/en/user-guide/data-load-snowpipe-rest-gs.html#client-requirement-java-or-python-sdk)
-2. [Snowpipe Streaming](https://docs.snowflake.com/en/LIMITEDACCESS/snowpipe-streaming.html) - Under Private Preview
+2. [Snowpipe Streaming](https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview) - Under Public Preview
 
 # Prerequisites
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
index 469a7a72c..ce576372e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
@@ -11,11 +11,15 @@
 
 /**
  * A logical partition that represents a connection to a single Snowflake table, data will be
- * ingested into the channel, and then flush to Snowflake table periodically in the background. Note
- * that only one client (or thread) could write to a channel at a given time, so if there are
- * multiple clients (or multiple threads in the same client) try to ingest using the same channel,
- * the latest client (or thread) that opens the channel will win and all the other opened channels
- * will be invalid
+ * ingested into the channel, and then flushed to Snowflake table periodically in the background.
+ *
+ * <p>Channels are identified by their name and only one channel with the same name may ingest data
+ * at the same time. When a new channel is opened, all previously opened channels with the same name
+ * are invalidated (this applies for the table globally. not just in a single JVM). In order to
+ * ingest data from multiple threads/clients/applications, we recommend opening multiple channels,
+ * each with a different name. There is no limit on the number of channels that can be opened.
+ *
+ * <p>Thread safety note: Implementations of this interface are required to be thread safe.
  */
 public interface SnowflakeStreamingIngestChannel {
   /**
@@ -220,7 +224,8 @@ public interface SnowflakeStreamingIngestChannel {
    *
    * </table>
    *
-   * @param row object data to write
+   * @param row object data to write. For predictable results, we recommend not to concurrently
+   *     modify the input row data.
    * @param offsetToken offset of given row, used for replay in case of failures. It could be null
    *     if you don't plan on replaying or can't replay
    * @return insert response that possibly contains errors because of insertion failures
diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
index fec3cd147..d5f9e5c2e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
@@ -9,6 +9,8 @@
  * maps to exactly one account in Snowflake; however, multiple clients can point to the same
  * account. Each client will contain information for Snowflake authentication and authorization, and
  * it will be used to create one or more {@link SnowflakeStreamingIngestChannel}
+ *
+ * <p>Thread safety note: Implementations of this interface are required to be thread safe.
  */
 public interface SnowflakeStreamingIngestClient extends AutoCloseable {
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 73adc2e4e..d9fd77273 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -386,13 +386,12 @@ private static OffsetDateTime inputToOffsetDateTime(
       }
 
       // Couldn't parse anything, throw an exception
-      // TODO Change URL when out of private preview
       throw valueFormatNotAllowedException(
           columnName,
           input.toString(),
           typeName,
           "Not a valid value, see"
-              + " https://docs.snowflake.com/en/LIMITEDACCESS/snowpipe-streaming.html"
+              + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview"
               + " for the list of supported formats");
     }
 
@@ -657,13 +656,12 @@ static BigInteger validateAndParseTime(String columnName, Object input, int scal
         }
       }
 
-      // TODO Change URL when out of private preview
       throw valueFormatNotAllowedException(
           columnName,
           input,
           "TIME",
           "Not a valid time, see"
-              + " https://docs.snowflake.com/en/LIMITEDACCESS/snowpipe-streaming.html"
+              + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview"
               + " for the list of supported formats");
 
     } else {
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 58948bd17..55e3298d9 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -917,7 +917,7 @@ public void testExceptionMessages() {
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
             + " Snowflake column COL of type TIME: Not a valid time, see"
-            + " https://docs.snowflake.com/en/LIMITEDACCESS/snowpipe-streaming.html"
+            + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview"
             + " for the list of supported formats",
         () -> validateAndParseTime("COL", "abc", 10));
 
@@ -932,8 +932,8 @@ public void testExceptionMessages() {
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
             + " Snowflake column COL of type DATE: Not a valid value, see"
-            + " https://docs.snowflake.com/en/LIMITEDACCESS/snowpipe-streaming.html for the list of"
-            + " supported formats",
+            + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
+            + " the list of supported formats",
         () -> validateAndParseDate("COL", "abc"));
 
     // TIMESTAMP_NTZ
@@ -947,8 +947,8 @@ public void testExceptionMessages() {
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
             + " Snowflake column COL of type TIMESTAMP: Not a valid value, see"
-            + " https://docs.snowflake.com/en/LIMITEDACCESS/snowpipe-streaming.html for the list of"
-            + " supported formats",
+            + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
+            + " the list of supported formats",
         () -> validateAndParseTimestamp("COL", "abc", 3, UTC, true));
 
     // TIMESTAMP_LTZ
@@ -962,8 +962,8 @@ public void testExceptionMessages() {
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
             + " Snowflake column COL of type TIMESTAMP: Not a valid value, see"
-            + " https://docs.snowflake.com/en/LIMITEDACCESS/snowpipe-streaming.html for the list of"
-            + " supported formats",
+            + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
+            + " the list of supported formats",
         () -> validateAndParseTimestamp("COL", "abc", 3, UTC, false));
 
     // TIMESTAMP_TZ
@@ -977,8 +977,8 @@ public void testExceptionMessages() {
         ErrorCode.INVALID_ROW,
         "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
             + " Snowflake column COL of type TIMESTAMP: Not a valid value, see"
-            + " https://docs.snowflake.com/en/LIMITEDACCESS/snowpipe-streaming.html for the list of"
-            + " supported formats",
+            + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
+            + " the list of supported formats",
         () -> validateAndParseTimestamp("COL", "abc", 3, UTC, false));
 
     // NUMBER

From 6cebb3fd6a72e1e489176a25332e4ada97217f31 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Mon, 6 Mar 2023 13:21:00 -0800
Subject: [PATCH 136/356] remove extra info and reduce to debug

---
 .../net/snowflake/ingest/connection/SecurityManager.java | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/connection/SecurityManager.java b/src/main/java/net/snowflake/ingest/connection/SecurityManager.java
index 581418479..96bc15e59 100644
--- a/src/main/java/net/snowflake/ingest/connection/SecurityManager.java
+++ b/src/main/java/net/snowflake/ingest/connection/SecurityManager.java
@@ -154,14 +154,12 @@ private void regenerateToken() {
     // create our JWT claim builder object
     JWTClaimsSet.Builder builder = new JWTClaimsSet.Builder();
 
-    // set the subject to the fully qualified username
+    // get the subject to the fully qualified username
     String subject = String.format("%s.%s", account, user);
-    LOGGER.info("Creating Token with subject {}", subject);
 
-    // set the issuer
+    // get the issuer
     String publicKeyFPInJwt = calculatePublicKeyFp(keyPair);
     String issuer = String.format("%s.%s.%s", account, user, publicKeyFPInJwt);
-    LOGGER.info("Creating Token with issuer {}", issuer);
 
     // iat set to now
     Date iat = new Date(System.currentTimeMillis());
@@ -172,6 +170,7 @@ private void regenerateToken() {
     // build claim set
     JWTClaimsSet claimsSet =
         builder.issuer(issuer).subject(subject).issueTime(iat).expirationTime(exp).build();
+    LOGGER.debug("Creating new JWT with subject {} and issuer {}...", subject, issuer);
 
     SignedJWT signedJWT = new SignedJWT(new JWSHeader(JWSAlgorithm.RS256), claimsSet);
 
@@ -188,7 +187,7 @@ private void regenerateToken() {
     }
 
     // atomically update the string
-    LOGGER.info("Created new JWT");
+    LOGGER.info("Successfully created new JWT");
     token.set(newToken);
 
     // Refresh the token used in the telemetry service as well

From 384cf69a9d5d1a5a20fa1606910ee1fdd7d033e5 Mon Sep 17 00:00:00 2001
From: sfc-gh-snyk-sca-sa
 <95290361+sfc-gh-snyk-sca-sa@users.noreply.github.com>
Date: Tue, 14 Mar 2023 06:17:23 +0530
Subject: [PATCH 137/356] SNOW-470176: [Snyk] Security upgrade
 com.fasterxml.jackson.core:jackson-databind from 2.14.0-rc1 to 2.14.0 (#347)

fix: pom.xml to reduce vulnerabilities

The following vulnerabilities are fixed with an upgrade:
- https://snyk.io/vuln/SNYK-JAVA-COMFASTERXMLJACKSONCORE-2421244

Co-authored-by: snyk-bot <snyk-bot@snyk.io>
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 7048aae29..322a4a464 100644
--- a/pom.xml
+++ b/pom.xml
@@ -208,7 +208,7 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.14.0-rc1</version>
+            <version>2.14.0</version>
         </dependency>
 
         <!-- SLF4J api that a client can shim in later -->

From 479b496984489b8adbc3f371167bf504166fbc52 Mon Sep 17 00:00:00 2001
From: David Wang <david.wang@snowflake.com>
Date: Thu, 16 Mar 2023 18:48:42 +0000
Subject: [PATCH 138/356] SNOW-761399: exclude nimbus-jose-jwt from
 hadoop-common dependency

---
 pom.xml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pom.xml b/pom.xml
index 322a4a464..83b115f71 100644
--- a/pom.xml
+++ b/pom.xml
@@ -334,6 +334,10 @@
                     <groupId>com.github.spotbugs</groupId>
                     <artifactId>spotbugs-annotations</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.nimbusds</groupId>
+                    <artifactId>nimbus-jose-jwt</artifactId>
+                </exclusion>
                 <exclusion>
                     <groupId>com.sun.jersey</groupId>
                     <artifactId>jersey-core</artifactId>

From f0e54f3624cc13d0651f5ed9a4638bf173e75170 Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Fri, 17 Mar 2023 13:30:14 +0100
Subject: [PATCH 139/356] @snow SNOW-759764 Streaming Ingest Parquet: use
 Snowflake column ordinal

---
 .../streaming/internal/ColumnMetadata.java    |  15 ++
 .../streaming/internal/ParquetRowBuffer.java  |   3 -
 .../internal/ParquetTypeGenerator.java        |   3 +-
 .../internal/ColumnMetadataBuilder.java       |  14 ++
 .../internal/ParquetTypeGeneratorTest.java    |  86 ++++++------
 .../streaming/internal/StreamingIngestIT.java | 130 +++++++++++++++++-
 6 files changed, 200 insertions(+), 51 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ColumnMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/ColumnMetadata.java
index b9779b41f..73b75da9e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ColumnMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ColumnMetadata.java
@@ -22,6 +22,12 @@ class ColumnMetadata {
   private boolean nullable;
   private String collation;
 
+  /**
+   * The column ordinal is an internal id of the column used by server scanner for the column
+   * identification.
+   */
+  private Integer ordinal;
+
   @JsonProperty("name")
   void setName(String name) {
     this.name = name;
@@ -113,6 +119,15 @@ boolean getNullable() {
     return this.nullable;
   }
 
+  @JsonProperty("ordinal")
+  void setOrdinal(Integer ordinal) {
+    this.ordinal = ordinal;
+  }
+
+  public Integer getOrdinal() {
+    return ordinal;
+  }
+
   String getInternalName() {
     return internalName;
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 46a756b60..e3ea8d89d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -90,9 +90,6 @@ public void setupSchema(List<ColumnMetadata> columns) {
     metadata.clear();
     metadata.put("sfVer", "1,1");
     List<Type> parquetTypes = new ArrayList<>();
-    // Snowflake column id that corresponds to the order in 'columns' received from server
-    // id is required to pack column metadata for the server scanner, e.g. decimal scale and
-    // precision
     int id = 1;
     for (ColumnMetadata column : columns) {
       validateColumnCollation(column);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
index 933909c2b..79da87241 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
@@ -60,10 +60,11 @@ public void setMetadata(Map<String, String> metadata) {
    * side.
    *
    * @param column column metadata as received from server side
-   * @param id column id
+   * @param id column id if column.getOrdinal() is not available
    * @return column parquet type
    */
   static ParquetTypeInfo generateColumnParquetTypeInfo(ColumnMetadata column, int id) {
+    id = column.getOrdinal() == null ? id : column.getOrdinal();
     ParquetTypeInfo res = new ParquetTypeInfo();
     Type parquetType;
     Map<String, String> metadata = new HashMap<>();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java b/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
index 51d4226bb..01fd44918 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ColumnMetadataBuilder.java
@@ -17,6 +17,8 @@ public class ColumnMetadataBuilder {
   private boolean nullable;
   private String collation;
 
+  private Integer ordinal;
+
   /**
    * Returns a new ColumnMetadata builder
    *
@@ -142,6 +144,17 @@ public ColumnMetadataBuilder collation(String collation) {
     return this;
   }
 
+  /**
+   * Set column ordinal
+   *
+   * @param ordinal ordinal
+   * @return columnMetadataBuilder object
+   */
+  public ColumnMetadataBuilder ordinal(int ordinal) {
+    this.ordinal = ordinal;
+    return this;
+  }
+
   /**
    * Build a columnMetadata object from the set values
    *
@@ -159,6 +172,7 @@ public ColumnMetadata build() {
     colMetadata.setScale(scale);
     colMetadata.setPrecision(precision);
     colMetadata.setCollation(collation);
+    colMetadata.setOrdinal(ordinal);
     return colMetadata;
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
index 0f6bf60c7..3356ab277 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetTypeGeneratorTest.java
@@ -8,11 +8,12 @@
 import org.junit.Test;
 
 public class ParquetTypeGeneratorTest {
+  private static final int COL_ORDINAL = 11;
 
   @Test
   public void buildFieldFixedSB1() {
     ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
+        createColumnMetadataBuilder()
             .logicalType("FIXED")
             .physicalType("SB1")
             .nullable(true)
@@ -20,10 +21,9 @@ public void buildFieldFixedSB1() {
 
     ParquetTypeGenerator.ParquetTypeInfo typeInfo =
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
+    createParquetTypeInfoAssertionBuilder()
         .typeInfo(typeInfo)
         .expectedFieldName("TESTCOL")
-        .expectedFieldId(0)
         .expectedTypeLength(4)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
         .expectedLogicalTypeAnnotation(
@@ -39,7 +39,7 @@ public void buildFieldFixedSB1() {
   @Test
   public void buildFieldFixedSB2() {
     ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
+        createColumnMetadataBuilder()
             .logicalType("FIXED")
             .physicalType("SB2")
             .nullable(false)
@@ -47,10 +47,9 @@ public void buildFieldFixedSB2() {
 
     ParquetTypeGenerator.ParquetTypeInfo typeInfo =
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
+    createParquetTypeInfoAssertionBuilder()
         .typeInfo(typeInfo)
         .expectedFieldName("TESTCOL")
-        .expectedFieldId(0)
         .expectedTypeLength(4)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
         .expectedLogicalTypeAnnotation(
@@ -66,7 +65,7 @@ public void buildFieldFixedSB2() {
   @Test
   public void buildFieldFixedSB4() {
     ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
+        createColumnMetadataBuilder()
             .logicalType("FIXED")
             .physicalType("SB4")
             .nullable(true)
@@ -74,10 +73,9 @@ public void buildFieldFixedSB4() {
 
     ParquetTypeGenerator.ParquetTypeInfo typeInfo =
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
+    createParquetTypeInfoAssertionBuilder()
         .typeInfo(typeInfo)
         .expectedFieldName("TESTCOL")
-        .expectedFieldId(0)
         .expectedTypeLength(4)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
         .expectedLogicalTypeAnnotation(
@@ -93,7 +91,7 @@ public void buildFieldFixedSB4() {
   @Test
   public void buildFieldFixedSB8() {
     ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
+        createColumnMetadataBuilder()
             .logicalType("FIXED")
             .physicalType("SB8")
             .nullable(true)
@@ -101,10 +99,9 @@ public void buildFieldFixedSB8() {
 
     ParquetTypeGenerator.ParquetTypeInfo typeInfo =
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
+    createParquetTypeInfoAssertionBuilder()
         .typeInfo(typeInfo)
         .expectedFieldName("TESTCOL")
-        .expectedFieldId(0)
         .expectedTypeLength(8)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
         .expectedLogicalTypeAnnotation(
@@ -120,7 +117,7 @@ public void buildFieldFixedSB8() {
   @Test
   public void buildFieldFixedSB16() {
     ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
+        createColumnMetadataBuilder()
             .logicalType("FIXED")
             .physicalType("SB16")
             .nullable(true)
@@ -128,10 +125,9 @@ public void buildFieldFixedSB16() {
 
     ParquetTypeGenerator.ParquetTypeInfo typeInfo =
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
+    createParquetTypeInfoAssertionBuilder()
         .typeInfo(typeInfo)
         .expectedFieldName("TESTCOL")
-        .expectedFieldId(0)
         .expectedTypeLength(16)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
         .expectedLogicalTypeAnnotation(
@@ -147,7 +143,7 @@ public void buildFieldFixedSB16() {
   @Test
   public void buildFieldLobVariant() {
     ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
+        createColumnMetadataBuilder()
             .logicalType("VARIANT")
             .physicalType("LOB")
             .nullable(true)
@@ -155,10 +151,9 @@ public void buildFieldLobVariant() {
 
     ParquetTypeGenerator.ParquetTypeInfo typeInfo =
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
+    createParquetTypeInfoAssertionBuilder()
         .typeInfo(typeInfo)
         .expectedFieldName("TESTCOL")
-        .expectedFieldId(0)
         .expectedTypeLength(0)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.BINARY)
         .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.stringType())
@@ -169,13 +164,13 @@ public void buildFieldLobVariant() {
                 + AbstractRowBuffer.ColumnPhysicalType.LOB.getOrdinal())
         .assertMatches();
 
-    Assert.assertEquals("1", typeInfo.getMetadata().get(0 + ":obj_enc"));
+    Assert.assertEquals("1", typeInfo.getMetadata().get(COL_ORDINAL + ":obj_enc"));
   }
 
   @Test
   public void buildFieldTimestampNtzSB8() {
     ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
+        createColumnMetadataBuilder()
             .logicalType("TIMESTAMP_NTZ")
             .physicalType("SB8")
             .nullable(true)
@@ -183,10 +178,9 @@ public void buildFieldTimestampNtzSB8() {
 
     ParquetTypeGenerator.ParquetTypeInfo typeInfo =
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
+    createParquetTypeInfoAssertionBuilder()
         .typeInfo(typeInfo)
         .expectedFieldName("TESTCOL")
-        .expectedFieldId(0)
         .expectedTypeLength(8)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
         .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
@@ -201,7 +195,7 @@ public void buildFieldTimestampNtzSB8() {
   @Test
   public void buildFieldTimestampNtzSB16() {
     ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
+        createColumnMetadataBuilder()
             .logicalType("TIMESTAMP_NTZ")
             .physicalType("SB16")
             .nullable(true)
@@ -209,10 +203,9 @@ public void buildFieldTimestampNtzSB16() {
 
     ParquetTypeGenerator.ParquetTypeInfo typeInfo =
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
+    createParquetTypeInfoAssertionBuilder()
         .typeInfo(typeInfo)
         .expectedFieldName("TESTCOL")
-        .expectedFieldId(0)
         .expectedTypeLength(16)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
         .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 38))
@@ -227,7 +220,7 @@ public void buildFieldTimestampNtzSB16() {
   @Test
   public void buildFieldTimestampTzSB8() {
     ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
+        createColumnMetadataBuilder()
             .logicalType("TIMESTAMP_TZ")
             .physicalType("SB8")
             .nullable(true)
@@ -235,10 +228,9 @@ public void buildFieldTimestampTzSB8() {
 
     ParquetTypeGenerator.ParquetTypeInfo typeInfo =
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
+    createParquetTypeInfoAssertionBuilder()
         .typeInfo(typeInfo)
         .expectedFieldName("TESTCOL")
-        .expectedFieldId(0)
         .expectedTypeLength(8)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
         .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
@@ -253,7 +245,7 @@ public void buildFieldTimestampTzSB8() {
   @Test
   public void buildFieldTimestampTzSB16() {
     ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
+        createColumnMetadataBuilder()
             .logicalType("TIMESTAMP_TZ")
             .physicalType("SB16")
             .nullable(true)
@@ -261,10 +253,9 @@ public void buildFieldTimestampTzSB16() {
 
     ParquetTypeGenerator.ParquetTypeInfo typeInfo =
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
+    createParquetTypeInfoAssertionBuilder()
         .typeInfo(typeInfo)
         .expectedFieldName("TESTCOL")
-        .expectedFieldId(0)
         .expectedTypeLength(16)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
         .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 38))
@@ -279,7 +270,7 @@ public void buildFieldTimestampTzSB16() {
   @Test
   public void buildFieldDate() {
     ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
+        createColumnMetadataBuilder()
             .logicalType("DATE")
             .physicalType("SB8")
             .nullable(true)
@@ -287,10 +278,9 @@ public void buildFieldDate() {
 
     ParquetTypeGenerator.ParquetTypeInfo typeInfo =
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
+    createParquetTypeInfoAssertionBuilder()
         .typeInfo(typeInfo)
         .expectedFieldName("TESTCOL")
-        .expectedFieldId(0)
         .expectedTypeLength(0)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
         .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.dateType())
@@ -305,7 +295,7 @@ public void buildFieldDate() {
   @Test
   public void buildFieldTimeSB4() {
     ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
+        createColumnMetadataBuilder()
             .logicalType("TIME")
             .physicalType("SB4")
             .nullable(true)
@@ -313,10 +303,9 @@ public void buildFieldTimeSB4() {
 
     ParquetTypeGenerator.ParquetTypeInfo typeInfo =
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
+    createParquetTypeInfoAssertionBuilder()
         .typeInfo(typeInfo)
         .expectedFieldName("TESTCOL")
-        .expectedFieldId(0)
         .expectedTypeLength(4)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT32)
         .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 9))
@@ -331,7 +320,7 @@ public void buildFieldTimeSB4() {
   @Test
   public void buildFieldTimeSB8() {
     ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
+        createColumnMetadataBuilder()
             .logicalType("TIME")
             .physicalType("SB8")
             .nullable(true)
@@ -339,10 +328,9 @@ public void buildFieldTimeSB8() {
 
     ParquetTypeGenerator.ParquetTypeInfo typeInfo =
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
+    createParquetTypeInfoAssertionBuilder()
         .typeInfo(typeInfo)
         .expectedFieldName("TESTCOL")
-        .expectedFieldId(0)
         .expectedTypeLength(8)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.INT64)
         .expectedLogicalTypeAnnotation(LogicalTypeAnnotation.decimalType(testCol.getScale(), 18))
@@ -357,7 +345,7 @@ public void buildFieldTimeSB8() {
   @Test
   public void buildFieldBoolean() {
     ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
+        createColumnMetadataBuilder()
             .logicalType("BOOLEAN")
             .physicalType("BINARY")
             .nullable(true)
@@ -365,10 +353,9 @@ public void buildFieldBoolean() {
 
     ParquetTypeGenerator.ParquetTypeInfo typeInfo =
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
+    createParquetTypeInfoAssertionBuilder()
         .typeInfo(typeInfo)
         .expectedFieldName("TESTCOL")
-        .expectedFieldId(0)
         .expectedTypeLength(0)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.BOOLEAN)
         .expectedLogicalTypeAnnotation(null)
@@ -383,7 +370,7 @@ public void buildFieldBoolean() {
   @Test
   public void buildFieldRealSB16() {
     ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
+        createColumnMetadataBuilder()
             .logicalType("REAL")
             .physicalType("SB16")
             .nullable(true)
@@ -391,10 +378,9 @@ public void buildFieldRealSB16() {
 
     ParquetTypeGenerator.ParquetTypeInfo typeInfo =
         ParquetTypeGenerator.generateColumnParquetTypeInfo(testCol, 0);
-    ParquetTypeInfoAssertionBuilder.newBuilder()
+    createParquetTypeInfoAssertionBuilder()
         .typeInfo(typeInfo)
         .expectedFieldName("TESTCOL")
-        .expectedFieldId(0)
         .expectedTypeLength(0)
         .expectedPrimitiveTypeName(PrimitiveType.PrimitiveTypeName.DOUBLE)
         .expectedLogicalTypeAnnotation(null)
@@ -477,4 +463,12 @@ void assertMatches() {
       Assert.assertEquals(colMetadata, metadata.get(type.getId().toString()));
     }
   }
+
+  private static ColumnMetadataBuilder createColumnMetadataBuilder() {
+    return ColumnMetadataBuilder.newBuilder().ordinal(COL_ORDINAL);
+  }
+
+  private static ParquetTypeInfoAssertionBuilder createParquetTypeInfoAssertionBuilder() {
+    return ParquetTypeInfoAssertionBuilder.newBuilder().expectedFieldId(COL_ORDINAL);
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 8c936bac4..052fc021f 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -3,6 +3,8 @@
 import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
 import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
 import static net.snowflake.ingest.utils.Constants.ROLE;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -39,6 +41,7 @@
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -994,6 +997,131 @@ public void testColumnNameQuotes() throws Exception {
     Assert.fail("Row sequencer not updated before timeout");
   }
 
+  @Ignore
+  @Test
+  public void testTableColumnEvolution() throws Exception {
+    final int rowNum = 100;
+    final String tableName = TEST_TABLE + "_TEST_COL_EVOL";
+    final String tableFullName = String.format("%s.%s.%s", testDb, TEST_SCHEMA, tableName);
+    final String streamName =
+        String.format("%s.%s.%s", testDb, TEST_SCHEMA, "STREAM_TEST_COL_EVOL");
+
+    executeQuery("create or replace table %s (c1 varchar, c2 varchar, c3 varchar);", tableName);
+    int channelNum = 1;
+
+    // insert first 3 columns
+    ingestByColIndexAndVerifyTable(tableName, channelNum, Arrays.asList(1, 2, 3), rowNum);
+    channelNum++;
+
+    // add one column
+    executeQuery(String.format("alter table %s add column c4 varchar;", tableFullName));
+    ingestByColIndexAndVerifyTable(tableName, channelNum, Arrays.asList(1, 2, 3, 4), rowNum);
+    channelNum++;
+
+    // remove first column
+    executeQuery(String.format("alter table %s drop column c1;", tableFullName));
+    ingestByColIndexAndVerifyTable(tableName, channelNum, Arrays.asList(2, 3, 4), rowNum);
+    channelNum++;
+
+    // remove last column
+    executeQuery(String.format("alter table %s drop column c4;", tableFullName));
+    ingestByColIndexAndVerifyTable(tableName, channelNum, Arrays.asList(2, 3), rowNum);
+    channelNum++;
+
+    // add last column
+    executeQuery(String.format("alter table %s add column c5 varchar;", tableFullName));
+    ingestByColIndexAndVerifyTable(tableName, channelNum, Arrays.asList(2, 3, 5), rowNum);
+    channelNum++;
+
+    // create stream on table to append change tracking columns
+    executeQuery(String.format("create or replace stream %s on table %s;", streamName, tableName));
+    ingestByColIndexAndVerifyTable(tableName, channelNum, Arrays.asList(2, 3, 5), rowNum);
+    channelNum++;
+
+    // add one more column after change tracking columns
+    executeQuery(String.format("alter table %s add column c6 varchar;", tableFullName));
+    ingestByColIndexAndVerifyTable(tableName, channelNum, Arrays.asList(2, 3, 5, 6), rowNum);
+  }
+
+  /**
+   * Ingests rows into a table and verifies them by querying the table.
+   *
+   * <p>The test opens channel 'channel<channelNum>' for ingestion.
+   *
+   * <p>Expected table schema is ('C<index1>' varchar, 'C<index2>' varchar) where indexes are from
+   * {@code columnIndexes}.
+   *
+   * <p>The ingested text values have the format: 'v<col_index1>_<row_index>'. The row index is from
+   * range [(channelNum - 1) * numberOfRows, numberOfRows].
+   *
+   * @param tableName table full name
+   * @param channelNum channel number to open
+   * @param columnIndexes indexes of columns to ingest
+   * @param numberOfRows to ingest
+   */
+  private void ingestByColIndexAndVerifyTable(
+      String tableName, int channelNum, List<Integer> columnIndexes, int numberOfRows)
+      throws InterruptedException, SQLException {
+    final String valueFormat = "v%01d_%03d";
+    final String channelName = "channel" + channelNum;
+    final int startIndex = (channelNum - 1) * numberOfRows;
+
+    // ingest rows
+    SnowflakeStreamingIngestChannel channel = openChannel(tableName, channelName);
+    for (int i = 0; i < numberOfRows; i++) {
+      Map<String, Object> row = new HashMap<>();
+      for (int col : columnIndexes) {
+        String value = String.format(valueFormat, col, startIndex + i);
+        row.put("c" + col, value);
+      }
+      verifyInsertValidationResponse(channel.insertRow(row, Integer.toString(i)));
+    }
+    TestUtils.waitForOffset(channel, Integer.toString(numberOfRows - 1));
+    channel.close();
+
+    // query them and verify
+    int firstColNum = columnIndexes.get(0);
+    try (ResultSet result =
+        jdbcConnection
+            .createStatement()
+            .executeQuery(
+                String.format(
+                    "select * from %s.%s.%s where c%d >= 'v%01d_%03d' and c%d < 'v%01d_%03d' order"
+                        + " by c%d",
+                    testDb,
+                    TEST_SCHEMA,
+                    tableName,
+                    firstColNum,
+                    firstColNum,
+                    startIndex,
+                    firstColNum,
+                    firstColNum,
+                    startIndex + numberOfRows,
+                    firstColNum))) {
+      for (int i = 0; i < numberOfRows; i++) {
+        result.next();
+        for (int col : columnIndexes) {
+          String value = result.getString("C" + col);
+          String expectedValue = String.format(valueFormat, col, startIndex + i);
+          String errorMessage =
+              String.format(
+                  "Ingested value mismatch: table %s, startIndex %d, column %s, index %d",
+                  tableName, startIndex, "c" + col, i);
+          assertThat(errorMessage, value, is(expectedValue));
+        }
+      }
+    }
+  }
+
+  private void executeQuery(String sqlFmt, Object... args) {
+    String sql = String.format(sqlFmt, args);
+    try {
+      jdbcConnection.createStatement().executeQuery(sql);
+    } catch (SQLException e) {
+      throw new RuntimeException("SQL query failed : " + sql, e);
+    }
+  }
+
   /** Verify the insert validation response and throw the exception if needed */
   private void verifyInsertValidationResponse(InsertValidationResponse response) {
     if (response.hasErrors()) {
@@ -1019,7 +1147,7 @@ private SnowflakeStreamingIngestChannel openChannel(String tableName, String cha
             .setDBName(testDb)
             .setSchemaName(TEST_SCHEMA)
             .setTableName(tableName)
-            .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+            .setOnErrorOption(OpenChannelRequest.OnErrorOption.ABORT)
             .build();
 
     // Open a streaming ingest channel from the given client

From e343fca41580d3443a462fcee38ebfec90872d68 Mon Sep 17 00:00:00 2001
From: David Wang <david.wang@snowflake.com>
Date: Wed, 22 Mar 2023 00:33:22 +0000
Subject: [PATCH 140/356] SNOW-765525 add compiling checker and clean up
 dependency

---
 pom.xml | 401 +++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 356 insertions(+), 45 deletions(-)

diff --git a/pom.xml b/pom.xml
index 83b115f71..adf1e176c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -135,6 +135,56 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+              <groupId>org.apache.maven.plugins</groupId>
+              <artifactId>maven-dependency-plugin</artifactId>
+              <version>3.3.0</version>
+              <executions>
+                <execution>
+                  <id>analyze</id>
+                  <goals>
+                    <goal>analyze-only</goal>
+                  </goals>
+                  <configuration>
+                    <failOnWarning>true</failOnWarning>
+                    <ignoreNonCompile>true</ignoreNonCompile>
+                    <ignoredUnusedDeclaredDependencies>
+                      <ignoredUnusedDeclaredDependency>com.fasterxml.woodstox:woodstox-core</ignoredUnusedDeclaredDependency>
+                      <ignoredUnusedDeclaredDependency>commons-io:commons-io</ignoredUnusedDeclaredDependency>
+                      <ignoredUnusedDeclaredDependency>commons-logging:commons-logging</ignoredUnusedDeclaredDependency>
+                    </ignoredUnusedDeclaredDependencies>
+                  </configuration>
+                </execution>
+              </executions>
+            </plugin>
+            <!--
+            <plugin>
+              <groupId>com.github.ekryd.sortpom</groupId>
+              <artifactId>sortpom-maven-plugin</artifactId>
+              <version>3.0.1</version>
+              <configuration>
+                <createBackupFile>false</createBackupFile>
+               <expandEmptyElements>false</expandEmptyElements>
+               <indentSchemaLocation>true</indentSchemaLocation>
+               <sortDependencies>scope,groupId,artifactId</sortDependencies>
+               <sortDependencyExclusions>groupId,artifactId</sortDependencyExclusions>
+               <sortExecutions>true</sortExecutions>
+               <sortModules>true</sortModules>
+               <sortPlugins>groupId,artifactId</sortPlugins>
+               <sortProperties>true</sortProperties>
+               <verifyFail>stop</verifyFail>
+               <verifyFailOn>strict</verifyFailOn>
+             </configuration>
+             <executions>
+               <execution>
+                 <goals>
+                   <goal>verify</goal>
+                 </goals>
+                 <phase>validate</phase>
+               </execution>
+             </executions>
+           </plugin>
+           -->
         </plugins>
 
         <!-- disable default maven deploy plugin since we are using gpg:sign-and-deploy-file -->
@@ -225,6 +275,12 @@
             <artifactId>slf4j-simple</artifactId>
             <version>1.7.21</version>
             <scope>test</scope>
+            <exclusions>
+              <exclusion>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-api</artifactId>
+              </exclusion>
+            </exclusions>
         </dependency>
 
         <!-- JUnit so that we can make some basic unit tests -->
@@ -239,6 +295,12 @@
             <artifactId>powermock-module-junit4</artifactId>
             <version>2.0.2</version>
             <scope>test</scope>
+            <exclusions>
+              <exclusion>
+                <groupId>junit</groupId>
+                <artifactId>junit</artifactId>
+              </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.mockito</groupId>
@@ -251,12 +313,32 @@
             <artifactId>powermock-api-mockito2</artifactId>
             <version>2.0.2</version>
             <scope>test</scope>
+            <exclusions>
+              <exclusion>
+                <groupId>org.mockito</groupId>
+                <artifactId>mockito-core</artifactId>
+              </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.powermock</groupId>
             <artifactId>powermock-core</artifactId>
             <version>2.0.2</version>
             <scope>test</scope>
+            <exclusions>
+              <exclusion>
+                <groupId>org.objenesis</groupId>
+                <artifactId>objenesis</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>net.bytebuddy</groupId>
+                <artifactId>byte-buddy-agent</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>net.bytebuddy</groupId>
+                <artifactId>byte-buddy</artifactId>
+              </exclusion>
+            </exclusions>
         </dependency>
 
         <!-- Apache Arrow -->
@@ -264,12 +346,36 @@
             <groupId>org.apache.arrow</groupId>
             <artifactId>arrow-vector</artifactId>
             <version>${arrow.version}</version>
+            <exclusions>
+              <exclusion>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-core</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-api</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-annotations</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-databind</artifactId>
+              </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.arrow</groupId>
             <artifactId>arrow-memory-netty</artifactId>
             <version>${arrow.version}</version>
             <scope>runtime</scope>
+            <exclusions>
+              <exclusion>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-api</artifactId>
+              </exclusion>
+            </exclusions>
         </dependency>
 
         <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop -->
@@ -278,53 +384,17 @@
             <artifactId>parquet-hadoop</artifactId>
             <version>1.12.3</version>
             <exclusions>
-                <exclusion>
-                    <groupId>javax.annotation</groupId>
-                    <artifactId>javax.annotation-api</artifactId>
-                </exclusion>
+              <exclusion>
+                  <groupId>javax.annotation</groupId>
+                  <artifactId>javax.annotation-api</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-api</artifactId>
+              </exclusion>
             </exclusions>
         </dependency>
 
-        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-mapreduce-client-core</artifactId>
-            <version>${hadoop.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.apache.hadoop</groupId>
-                    <artifactId>hadoop-yarn-common</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.hadoop</groupId>
-                    <artifactId>hadoop-hdfs-client</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.google.protobuf</groupId>
-                    <artifactId>protobuf-java</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.avro</groupId>
-                    <artifactId>avro</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.google.inject.extensions</groupId>
-                    <artifactId>guice-servlet</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.eclipse.jetty.websocket</groupId>
-                    <artifactId>websocket-client</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>io.netty</groupId>
-                    <artifactId>netty</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-reload4j</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
@@ -402,6 +472,38 @@
                     <groupId>org.slf4j</groupId>
                     <artifactId>slf4j-reload4j</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.xerial.snappy</groupId>
+                    <artifactId>snappy-java</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.commons</groupId>
+                    <artifactId>commons-lang3</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-codec</groupId>
+                    <artifactId>commons-codec</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-logging</groupId>
+                    <artifactId>commons-logging</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>commons-io</groupId>
+                    <artifactId>commons-io</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-databind</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 
@@ -410,6 +512,12 @@
             <groupId>io.dropwizard.metrics</groupId>
             <artifactId>metrics-core</artifactId>
             <version>4.1.22</version>
+            <exclusions>
+              <exclusion>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-api</artifactId>
+              </exclusion>
+            </exclusions>
         </dependency>
 
         <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-jvm -->
@@ -417,13 +525,122 @@
             <groupId>io.dropwizard.metrics</groupId>
             <artifactId>metrics-jvm</artifactId>
             <version>4.1.22</version>
+            <exclusions>
+              <exclusion>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-api</artifactId>
+              </exclusion>
+            </exclusions>
         </dependency>
 
         <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-jmx -->
         <dependency>
             <groupId>io.dropwizard.metrics</groupId>
             <artifactId>metrics-jmx</artifactId>
-            <version>4.2.3</version>
+            <version>4.1.22</version>
+            <exclusions>
+              <exclusion>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-api</artifactId>
+              </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.code.findbugs</groupId>
+            <artifactId>jsr305</artifactId>
+            <version>3.0.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-common</artifactId>
+            <version>1.12.3</version>
+            <exclusions>
+              <exclusion>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-api</artifactId>
+              </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-annotations</artifactId>
+            <version>2.14.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+            <version>2.14.0</version>
+        </dependency>
+        <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-common</artifactId>
+            <version>4.1.82.Final</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+            <version>1.15</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-memory-core</artifactId>
+            <version>10.0.0</version>
+            <exclusions>
+              <exclusion>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-api</artifactId>
+              </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-column</artifactId>
+            <version>1.12.3</version>
+            <exclusions>
+              <exclusion>
+                  <groupId>org.slf4j</groupId>
+                  <artifactId>slf4j-api</artifactId>
+              </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.12.0</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId> org.hamcrest</groupId>
+            <artifactId>hamcrest-core</artifactId>
+            <version>1.3</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>31.1-jre</version>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.woodstox</groupId>
+            <artifactId>woodstox-core</artifactId>
+            <version>5.3.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.codehaus.woodstox</groupId>
+                    <artifactId>stax2-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+            <version>1.2</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>2.8.0</version>
         </dependency>
     </dependencies>
 
@@ -633,6 +850,100 @@
                 </plugins>
             </build>
         </profile>
+        <profile>
+          <id>run-enforcer</id>
+            <activation>
+              <activeByDefault>false</activeByDefault>
+            </activation>
+          <build>
+            <plugins>
+              <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-enforcer-plugin</artifactId>
+                <version>3.0.0-M3</version>
+                <dependencies>
+                  <dependency>
+                    <groupId>com.google.cloud.tools</groupId>
+                    <artifactId>linkage-checker-enforcer-rules</artifactId>
+                    <version>1.5.12</version>
+                  </dependency>
+                  <dependency>
+                    <groupId>org.codehaus.mojo</groupId>
+                    <artifactId>extra-enforcer-rules</artifactId>
+                    <version>1.3</version>
+                    <exclusions>
+                      <exclusion>
+                        <groupId>org.eclipse.aether</groupId>
+                        <artifactId>aether-util</artifactId>
+                      </exclusion>
+                    </exclusions>
+                  </dependency>
+                </dependencies>
+                <executions>
+                  <execution>
+                    <id>enforce-best-practices</id>
+                    <goals>
+                      <goal>enforce</goal>
+                    </goals>
+                    <configuration>
+                      <rules>
+                        <banDuplicateClasses>
+                          <findAllDuplicates>true</findAllDuplicates>
+                          <ignoreWhenIdentical>true</ignoreWhenIdentical>
+                          <ignoreClasses>
+                            <!-- <ignoreClass>com.snowflake.jmh.*</ignoreClass> -->
+                          </ignoreClasses>
+                        </banDuplicateClasses>
+                        <banDuplicatePomDependencyVersions/>
+                        <bannedDependencies>
+                          <excludes>
+                            <!-- <exclude>org.bouncycastle:bcprov-jdk15on</exclude> -->
+                          </excludes>
+                        </bannedDependencies>
+                        <dependencyConvergence/>
+                        <requireUpperBoundDeps/>
+                      </rules>
+                    </configuration>
+                  </execution>
+                  <execution>
+                    <id>enforce-maven</id>
+                    <goals>
+                      <goal>enforce</goal>
+                    </goals>
+                    <configuration>
+                      <rules>
+                        <requireMavenVersion>
+                        <version>3.5</version>
+                        </requireMavenVersion>
+                      </rules>
+                    </configuration>
+                  </execution>
+                  <execution>
+                    <id>enforce-linkage-checker</id>
+                    <goals>
+                      <goal>enforce</goal>
+                    </goals>
+                    <phase>verify</phase>
+                    <configuration>
+                      <rules>
+                        <LinkageCheckerRule implementation="com.google.cloud.tools.dependencies.enforcer.LinkageCheckerRule">
+                          <reportOnlyReachable>true</reportOnlyReachable>
+                          <!-- <exclusionFile>${basedir}/linkage-checker-exclusion-rules.xml</exclusionFile> -->
+                        </LinkageCheckerRule>
+                        <bannedDependencies>
+                          <excludes>
+                            <!-- JUnit 4 is not allowed on the classpath to prevent its accidental use -->
+                            <!-- <exclude>junit:junit</exclude> -->
+                          </excludes>
+                        </bannedDependencies>
+                      </rules>
+                    </configuration>
+                  </execution>
+                </executions>
+              </plugin>
+            </plugins>
+          </build>
+        </profile>
     </profiles>
 
 </project>

From eed893cafaa1a7ce1d5dfa6f9947104b8cdb68cf Mon Sep 17 00:00:00 2001
From: David Wang <david.wang@snowflake.com>
Date: Wed, 22 Mar 2023 19:38:51 +0000
Subject: [PATCH 141/356] SNOW-765525 update pom.xml to fix some check issue

---
 linkage-checker-exclusion-rules.xml |    8 +
 pom.xml                             | 1757 +++++++++++++--------------
 2 files changed, 842 insertions(+), 923 deletions(-)
 create mode 100644 linkage-checker-exclusion-rules.xml

diff --git a/linkage-checker-exclusion-rules.xml b/linkage-checker-exclusion-rules.xml
new file mode 100644
index 000000000..66e1455c3
--- /dev/null
+++ b/linkage-checker-exclusion-rules.xml
@@ -0,0 +1,8 @@
+<LinkageCheckerFilter>
+  <!-- <LinkageError>
+           <Source><Package name=""/></Source>
+    <Target><Package name=""/></Target>
+    <Reason></Reason>
+  </LinkageError> -->
+</LinkageCheckerFilter>
+
diff --git a/pom.xml b/pom.xml
index adf1e176c..2ba6bf1aa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,949 +1,860 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
 
-    <!-- Arifact name and version information -->
-    <groupId>net.snowflake</groupId>
-    <artifactId>snowflake-ingest-sdk</artifactId>
-    <version>1.1.0</version>
-    <packaging>jar</packaging>
-    <name>Snowflake Ingest SDK</name>
-    <description>Snowflake Ingest SDK</description>
-    <url>https://www.snowflake.net/</url>
+  <!-- Arifact name and version information -->
+  <groupId>net.snowflake</groupId>
+  <artifactId>snowflake-ingest-sdk</artifactId>
+  <version>1.1.0</version>
+  <packaging>jar</packaging>
+  <name>Snowflake Ingest SDK</name>
+  <description>Snowflake Ingest SDK</description>
+  <url>https://www.snowflake.net/</url>
 
+  <licenses>
+    <license>
+      <name>The Apache Software License, Version 2.0</name>
+      <url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
+    </license>
+  </licenses>
 
-    <licenses>
-      <license>
-        <name>The Apache Software License, Version 2.0</name>
-        <url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
-      </license>
-    </licenses>
+  <developers>
+    <developer>
+      <name>Snowflake Support Team</name>
+      <email>snowflake-java@snowflake.net</email>
+      <organization>Snowflake Computing</organization>
+      <organizationUrl>https://www.snowflake.net</organizationUrl>
+    </developer>
+  </developers>
 
-    <developers>
-      <developer>
-        <name>Snowflake Support Team</name>
-        <email>snowflake-java@snowflake.net</email>
-        <organization>Snowflake Computing</organization>
-        <organizationUrl>https://www.snowflake.net</organizationUrl>
-      </developer>
-    </developers>
+  <scm>
+    <connection>scm:git:git://github.com/snowflakedb/snowflake-ingest-java</connection>
+    <url>https://github.com/snowflakedb/snowflake-ingest-java/tree/master</url>
+  </scm>
 
-    <scm>
-      <connection>scm:git:git://github.com/snowflakedb/snowflake-ingest-java</connection>
-      <url>https://github.com/snowflakedb/snowflake-ingest-java/tree/master</url>
-    </scm>
+  <!-- Set our Language Level to Java 8 -->
+  <properties>
+    <arrow.version>10.0.0</arrow.version>
+    <hadoop.version>3.3.4</hadoop.version>
+    <jacoco.skip.instrument>true</jacoco.skip.instrument>
+    <jacoco.version>0.8.5</jacoco.version>
+    <maven.compiler.source>1.8</maven.compiler.source>
+    <maven.compiler.target>1.8</maven.compiler.target>
+    <shadeBase>net.snowflake.ingest.internal</shadeBase>
+  </properties>
 
-    <!-- Set our Language Level to Java 8 -->
-    <properties>
-        <maven.compiler.source>1.8</maven.compiler.source>
-        <maven.compiler.target>1.8</maven.compiler.target>
-        <shadeBase>net.snowflake.ingest.internal</shadeBase>
-        <jacoco.skip.instrument>true</jacoco.skip.instrument>
-        <jacoco.version>0.8.5</jacoco.version>
-        <arrow.version>10.0.0</arrow.version>
-        <hadoop.version>3.3.4</hadoop.version>
-    </properties>
-
-    <build>
-        <finalName>${project.artifactId}</finalName>
-        <resources>
-            <resource>
-                <directory>src/main/resources</directory>
-                <filtering>true</filtering>
-            </resource>
-        </resources>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.19.1</version>
-                <configuration>
-                    <excludes>
-                        <exclude>**/TestSimpleIngestLocal.java</exclude>
-                    </excludes>
-                </configuration>
-            </plugin>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <configuration>
-                    <archive>
-                        <manifest>
-                            <addClasspath>true</addClasspath>
-                            <mainClass>net.snowflake.ingest.SimpleIngestManager</mainClass>
-                        </manifest>
-                    </archive>
-                    <descriptorRefs>
-                        <descriptorRef>jar-with-dependencies</descriptorRef>
-                    </descriptorRefs>
-                </configuration>
-            </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-compiler-plugin</artifactId>
-                <version>2.5.1</version>
-                <inherited>true</inherited>
-                <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
-                </configuration>
-            </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-javadoc-plugin</artifactId>
-                <version>2.10.4</version>
-                <executions>
-                    <execution>
-                        <id>attach-javadocs</id>
-                        <goals>
-                            <goal>jar</goal>
-                        </goals>
-                        <configuration>
-                            <additionalparam>-Xdoclint:none</additionalparam>
-                            <source>8</source>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-source-plugin</artifactId>
-                <version>3.0.1</version>
-                <executions>
-                    <execution>
-                        <id>attach-sources</id>
-                        <goals>
-                            <goal>jar</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>exec-maven-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>check-shaded-content</id>
-                        <phase>verify</phase>
-                        <goals>
-                            <goal>exec</goal>
-                        </goals>
-                        <configuration>
-                            <executable>${basedir}/scripts/check_content.sh</executable>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-              <groupId>org.apache.maven.plugins</groupId>
-              <artifactId>maven-dependency-plugin</artifactId>
-              <version>3.3.0</version>
-              <executions>
-                <execution>
-                  <id>analyze</id>
-                  <goals>
-                    <goal>analyze-only</goal>
-                  </goals>
-                  <configuration>
-                    <failOnWarning>true</failOnWarning>
-                    <ignoreNonCompile>true</ignoreNonCompile>
-                    <ignoredUnusedDeclaredDependencies>
-                      <ignoredUnusedDeclaredDependency>com.fasterxml.woodstox:woodstox-core</ignoredUnusedDeclaredDependency>
-                      <ignoredUnusedDeclaredDependency>commons-io:commons-io</ignoredUnusedDeclaredDependency>
-                      <ignoredUnusedDeclaredDependency>commons-logging:commons-logging</ignoredUnusedDeclaredDependency>
-                    </ignoredUnusedDeclaredDependencies>
-                  </configuration>
-                </execution>
-              </executions>
-            </plugin>
-            <!--
-            <plugin>
-              <groupId>com.github.ekryd.sortpom</groupId>
-              <artifactId>sortpom-maven-plugin</artifactId>
-              <version>3.0.1</version>
-              <configuration>
-                <createBackupFile>false</createBackupFile>
-               <expandEmptyElements>false</expandEmptyElements>
-               <indentSchemaLocation>true</indentSchemaLocation>
-               <sortDependencies>scope,groupId,artifactId</sortDependencies>
-               <sortDependencyExclusions>groupId,artifactId</sortDependencyExclusions>
-               <sortExecutions>true</sortExecutions>
-               <sortModules>true</sortModules>
-               <sortPlugins>groupId,artifactId</sortPlugins>
-               <sortProperties>true</sortProperties>
-               <verifyFail>stop</verifyFail>
-               <verifyFailOn>strict</verifyFailOn>
-             </configuration>
-             <executions>
-               <execution>
-                 <goals>
-                   <goal>verify</goal>
-                 </goals>
-                 <phase>validate</phase>
-               </execution>
-             </executions>
-           </plugin>
-           -->
-        </plugins>
-
-        <!-- disable default maven deploy plugin since we are using gpg:sign-and-deploy-file -->
-        <pluginManagement>
-            <plugins>
-                <plugin>
-                    <artifactId>maven-deploy-plugin</artifactId>
-                    <configuration>
-                        <skip>true</skip>
-                    </configuration>
-                </plugin>
-                <plugin>
-                    <groupId>org.jacoco</groupId>
-                    <artifactId>jacoco-maven-plugin</artifactId>
-                    <version>${jacoco.version}</version>
-                    <executions>
-                        <execution>
-                            <id>pre-unit-test</id>
-                            <goals>
-                                <goal>prepare-agent</goal>
-                            </goals>
-                            <configuration>
-                                <destFile>target/jacoco-ut.exec</destFile>
-                            </configuration>
-                        </execution>
-                        <execution>
-                            <id>post-unit-test</id>
-                            <phase>test</phase>
-                            <goals>
-                                <goal>report</goal>
-                            </goals>
-                            <configuration>
-                                <dataFile>target/jacoco-ut.exec</dataFile>
-                                <outputDirectory>target/jacoco-ut</outputDirectory>
-                            </configuration>
-                        </execution>
-                    </executions>
-                    <configuration>
-                        <skip>${jacoco.skip.instrument}</skip>
-                    </configuration>
-                </plugin>
-                <!-- Need to define the maven-failsafe-plugin here -->
-                <!-- http://maven.apache.org/surefire/maven-failsafe-plugin/plugin-info.html -->
-                <plugin>
-                    <groupId>org.apache.maven.plugins</groupId>
-                    <artifactId>maven-failsafe-plugin</artifactId>
-                    <version>3.0.0-M5</version>
-                </plugin>
-            </plugins>
-        </pluginManagement>
-    </build>
-
-    <!-- All of our needed dependencies -->
+  <dependencyManagement>
     <dependencies>
+      <dependency>
+        <groupId>ch.qos.reload4j</groupId>
+        <artifactId>reload4j</artifactId>
+        <version>1.2.22</version>
+      </dependency>
+      <dependency>
+        <groupId>com.fasterxml.jackson.core</groupId>
+        <artifactId>jackson-annotations</artifactId>
+        <version>2.14.0</version>
+      </dependency>
+      <dependency>
+        <groupId>com.fasterxml.jackson.core</groupId>
+        <artifactId>jackson-core</artifactId>
+        <version>2.14.0</version>
+      </dependency>
+      <!-- Jackson Databind api -->
+      <dependency>
+        <groupId>com.fasterxml.jackson.core</groupId>
+        <artifactId>jackson-databind</artifactId>
+        <version>2.14.0</version>
+      </dependency>
+      <dependency>
+        <groupId>com.google.guava</groupId>
+        <artifactId>guava</artifactId>
+        <version>31.1-jre</version>
+      </dependency>
+      <dependency>
+        <groupId>com.nimbusds</groupId>
+        <artifactId>nimbus-jose-jwt</artifactId>
+        <version>9.9.3</version>
+      </dependency>
+      <dependency>
+        <groupId>commons-codec</groupId>
+        <artifactId>commons-codec</artifactId>
+        <version>1.15</version>
+      </dependency>
+      <dependency>
+        <groupId>commons-io</groupId>
+        <artifactId>commons-io</artifactId>
+        <version>2.8.0</version>
+      </dependency>
+      <dependency>
+        <groupId>commons-logging</groupId>
+        <artifactId>commons-logging</artifactId>
+        <version>1.2</version>
+      </dependency>
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-buffer</artifactId>
+        <version>4.1.82.Final</version>
+      </dependency>
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-common</artifactId>
+        <version>4.1.82.Final</version>
+      </dependency>
+      <dependency>
+        <groupId>net.snowflake</groupId>
+        <artifactId>snowflake-jdbc</artifactId>
+        <version>3.13.15</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.commons</groupId>
+        <artifactId>commons-compress</artifactId>
+        <version>1.21</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.commons</groupId>
+        <artifactId>commons-lang3</artifactId>
+        <version>3.12.0</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.yetus</groupId>
+        <artifactId>audience-annotations</artifactId>
+        <version>0.13.0</version>
+      </dependency>
+      <dependency>
+        <groupId>org.codehaus.jackson</groupId>
+        <artifactId>jackson-core-asl</artifactId>
+        <version>1.9.13</version>
+      </dependency>
+      <dependency>
+        <groupId>org.codehaus.jackson</groupId>
+        <artifactId>jackson-mapper-asl</artifactId>
+        <version>1.9.13</version>
+      </dependency>
+      <dependency>
+        <groupId>org.codehaus.woodstox</groupId>
+        <artifactId>stax2-api</artifactId>
+        <version>4.2.1</version>
+      </dependency>
+      <dependency>
+        <groupId>org.objenesis</groupId>
+        <artifactId>objenesis</artifactId>
+        <version>3.1</version>
+      </dependency>
+      <!-- SLF4J api that a client can shim in later -->
+      <dependency>
+        <groupId>org.slf4j</groupId>
+        <artifactId>slf4j-api</artifactId>
+        <version>1.7.36</version>
+      </dependency>
+      <!-- JDK logger backend for logging tests -->
+      <dependency>
+        <groupId>org.slf4j</groupId>
+        <artifactId>slf4j-simple</artifactId>
+        <version>1.7.36</version>
+      </dependency>
+      <dependency>
+        <groupId>org.xerial.snappy</groupId>
+        <artifactId>snappy-java</artifactId>
+        <version>1.1.8.3</version>
+      </dependency>
 
-        <!-- Snowflake JDBC used to connect to the service-->
-        <dependency>
-            <groupId>net.snowflake</groupId>
-            <artifactId>snowflake-jdbc</artifactId>
-            <version>3.13.15</version>
-        </dependency>
+      <!-- JUnit so that we can make some basic unit tests -->
+      <dependency>
+        <groupId>junit</groupId>
+        <artifactId>junit</artifactId>
+        <version>4.13.2</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>net.bytebuddy</groupId>
+        <artifactId>byte-buddy</artifactId>
+        <version>1.10.19</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>net.bytebuddy</groupId>
+        <artifactId>byte-buddy-agent</artifactId>
+        <version>1.10.19</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.mockito</groupId>
+        <artifactId>mockito-core</artifactId>
+        <version>3.7.7</version>
+        <scope>test</scope>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
 
-        <!-- jwt token for key pair authentication with GS -->
-        <dependency>
-            <groupId>com.nimbusds</groupId>
-            <artifactId>nimbus-jose-jwt</artifactId>
-            <version>9.9.3</version>
-         </dependency>
+  <!-- All of our needed dependencies -->
+  <dependencies>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-annotations</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
+    </dependency>
+    <!-- Jackson Databind api -->
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+    </dependency>
 
-        <!-- Jackson Databind api -->
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>2.14.0</version>
-        </dependency>
+    <dependency>
+      <groupId>com.google.code.findbugs</groupId>
+      <artifactId>jsr305</artifactId>
+      <version>3.0.2</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
 
-        <!-- SLF4J api that a client can shim in later -->
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.21</version>
-            <scope>provided</scope>
-        </dependency>
+    <!-- jwt token for key pair authentication with GS -->
+    <dependency>
+      <groupId>com.nimbusds</groupId>
+      <artifactId>nimbus-jose-jwt</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>commons-codec</groupId>
+      <artifactId>commons-codec</artifactId>
+    </dependency>
 
-        <!-- JDK logger backend for logging tests -->
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-simple</artifactId>
-            <version>1.7.21</version>
-            <scope>test</scope>
-            <exclusions>
-              <exclusion>
-                <groupId>org.slf4j</groupId>
-                <artifactId>slf4j-api</artifactId>
-              </exclusion>
-            </exclusions>
-        </dependency>
+    <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-core -->
+    <dependency>
+      <groupId>io.dropwizard.metrics</groupId>
+      <artifactId>metrics-core</artifactId>
+      <version>4.1.22</version>
+    </dependency>
 
-        <!-- JUnit so that we can make some basic unit tests -->
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.13.2</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.powermock</groupId>
-            <artifactId>powermock-module-junit4</artifactId>
-            <version>2.0.2</version>
-            <scope>test</scope>
-            <exclusions>
-              <exclusion>
-                <groupId>junit</groupId>
-                <artifactId>junit</artifactId>
-              </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>3.7.7</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.powermock</groupId>
-            <artifactId>powermock-api-mockito2</artifactId>
-            <version>2.0.2</version>
-            <scope>test</scope>
-            <exclusions>
-              <exclusion>
-                <groupId>org.mockito</groupId>
-                <artifactId>mockito-core</artifactId>
-              </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>org.powermock</groupId>
-            <artifactId>powermock-core</artifactId>
-            <version>2.0.2</version>
-            <scope>test</scope>
-            <exclusions>
-              <exclusion>
-                <groupId>org.objenesis</groupId>
-                <artifactId>objenesis</artifactId>
-              </exclusion>
-              <exclusion>
-                <groupId>net.bytebuddy</groupId>
-                <artifactId>byte-buddy-agent</artifactId>
-              </exclusion>
-              <exclusion>
-                <groupId>net.bytebuddy</groupId>
-                <artifactId>byte-buddy</artifactId>
-              </exclusion>
-            </exclusions>
-        </dependency>
+    <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-jmx -->
+    <dependency>
+      <groupId>io.dropwizard.metrics</groupId>
+      <artifactId>metrics-jmx</artifactId>
+      <version>4.1.22</version>
+    </dependency>
 
-        <!-- Apache Arrow -->
-        <dependency>
-            <groupId>org.apache.arrow</groupId>
-            <artifactId>arrow-vector</artifactId>
-            <version>${arrow.version}</version>
-            <exclusions>
-              <exclusion>
-                <groupId>com.fasterxml.jackson.core</groupId>
-                <artifactId>jackson-core</artifactId>
-              </exclusion>
-              <exclusion>
-                <groupId>org.slf4j</groupId>
-                <artifactId>slf4j-api</artifactId>
-              </exclusion>
-              <exclusion>
-                <groupId>com.fasterxml.jackson.core</groupId>
-                <artifactId>jackson-annotations</artifactId>
-              </exclusion>
-              <exclusion>
-                <groupId>com.fasterxml.jackson.core</groupId>
-                <artifactId>jackson-databind</artifactId>
-              </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.arrow</groupId>
-            <artifactId>arrow-memory-netty</artifactId>
-            <version>${arrow.version}</version>
-            <scope>runtime</scope>
-            <exclusions>
-              <exclusion>
-                <groupId>org.slf4j</groupId>
-                <artifactId>slf4j-api</artifactId>
-              </exclusion>
-            </exclusions>
-        </dependency>
+    <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-jvm -->
+    <dependency>
+      <groupId>io.dropwizard.metrics</groupId>
+      <artifactId>metrics-jvm</artifactId>
+      <version>4.1.22</version>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>net.bytebuddy</groupId>
+      <artifactId>byte-buddy-agent</artifactId>
+    </dependency>
 
-        <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop -->
-        <dependency>
-            <groupId>org.apache.parquet</groupId>
-            <artifactId>parquet-hadoop</artifactId>
-            <version>1.12.3</version>
-            <exclusions>
-              <exclusion>
-                  <groupId>javax.annotation</groupId>
-                  <artifactId>javax.annotation-api</artifactId>
-              </exclusion>
-              <exclusion>
-                <groupId>org.slf4j</groupId>
-                <artifactId>slf4j-api</artifactId>
-              </exclusion>
-            </exclusions>
-        </dependency>
+    <!-- Snowflake JDBC used to connect to the service-->
+    <dependency>
+      <groupId>net.snowflake</groupId>
+      <artifactId>snowflake-jdbc</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-memory-core</artifactId>
+      <version>${arrow.version}</version>
+    </dependency>
 
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-common</artifactId>
-            <version>${hadoop.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>com.github.spotbugs</groupId>
-                    <artifactId>spotbugs-annotations</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.nimbusds</groupId>
-                    <artifactId>nimbus-jose-jwt</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.sun.jersey</groupId>
-                    <artifactId>jersey-core</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.sun.jersey</groupId>
-                    <artifactId>jersey-json</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.sun.jersey</groupId>
-                    <artifactId>jersey-server</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.sun.jersey</groupId>
-                    <artifactId>jersey-servlet</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>javax.servlet</groupId>
-                    <artifactId>servlet-api</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>javax.servlet</groupId>
-                    <artifactId>javax.servlet-api</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>javax.servlet.jsp</groupId>
-                    <artifactId>jsp-api</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>log4j</groupId>
-                    <artifactId>log4j</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.codehaus.jackson</groupId>
-                    <artifactId>jackson-mapper-asl</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.google.protobuf</groupId>
-                    <artifactId>protobuf-java</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.mortbay.jetty</groupId>
-                    <artifactId>jetty</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.eclipse.jetty</groupId>
-                    <artifactId>jetty-server</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-log4j12</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.zookeeper</groupId>
-                    <artifactId>zookeeper</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.avro</groupId>
-                    <artifactId>avro</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-reload4j</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.google.guava</groupId>
-                    <artifactId>guava</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.xerial.snappy</groupId>
-                    <artifactId>snappy-java</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.commons</groupId>
-                    <artifactId>commons-lang3</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-api</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>commons-codec</groupId>
-                    <artifactId>commons-codec</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>commons-logging</groupId>
-                    <artifactId>commons-logging</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>commons-io</groupId>
-                    <artifactId>commons-io</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>jackson-databind</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
+    <!-- Apache Arrow -->
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-vector</artifactId>
+      <version>${arrow.version}</version>
+    </dependency>
 
-        <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-core -->
-        <dependency>
-            <groupId>io.dropwizard.metrics</groupId>
-            <artifactId>metrics-core</artifactId>
-            <version>4.1.22</version>
-            <exclusions>
-              <exclusion>
-                <groupId>org.slf4j</groupId>
-                <artifactId>slf4j-api</artifactId>
-              </exclusion>
-            </exclusions>
-        </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>${hadoop.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.parquet</groupId>
+      <artifactId>parquet-column</artifactId>
+      <version>1.12.3</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.parquet</groupId>
+      <artifactId>parquet-common</artifactId>
+      <version>1.12.3</version>
+    </dependency>
 
-        <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-jvm -->
-        <dependency>
-            <groupId>io.dropwizard.metrics</groupId>
-            <artifactId>metrics-jvm</artifactId>
-            <version>4.1.22</version>
-            <exclusions>
-              <exclusion>
-                <groupId>org.slf4j</groupId>
-                <artifactId>slf4j-api</artifactId>
-              </exclusion>
-            </exclusions>
-        </dependency>
+    <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop -->
+    <dependency>
+      <groupId>org.apache.parquet</groupId>
+      <artifactId>parquet-hadoop</artifactId>
+      <version>1.12.3</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-memory-netty</artifactId>
+      <version>${arrow.version}</version>
+      <scope>runtime</scope>
+    </dependency>
 
-        <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-jmx -->
-        <dependency>
-            <groupId>io.dropwizard.metrics</groupId>
-            <artifactId>metrics-jmx</artifactId>
-            <version>4.1.22</version>
-            <exclusions>
-              <exclusion>
-                <groupId>org.slf4j</groupId>
-                <artifactId>slf4j-api</artifactId>
-              </exclusion>
-            </exclusions>
-        </dependency>
+    <!-- JUnit so that we can make some basic unit tests -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.hamcrest</groupId>
+      <artifactId>hamcrest-core</artifactId>
+      <version>1.3</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.powermock</groupId>
+      <artifactId>powermock-api-mockito2</artifactId>
+      <version>2.0.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.powermock</groupId>
+      <artifactId>powermock-core</artifactId>
+      <version>2.0.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.powermock</groupId>
+      <artifactId>powermock-module-junit4</artifactId>
+      <version>2.0.2</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
 
-        <dependency>
-            <groupId>com.google.code.findbugs</groupId>
-            <artifactId>jsr305</artifactId>
-            <version>3.0.2</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.parquet</groupId>
-            <artifactId>parquet-common</artifactId>
-            <version>1.12.3</version>
-            <exclusions>
-              <exclusion>
-                <groupId>org.slf4j</groupId>
-                <artifactId>slf4j-api</artifactId>
-              </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-annotations</artifactId>
-            <version>2.14.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-core</artifactId>
-            <version>2.14.0</version>
-        </dependency>
-        <dependency>
-            <groupId>io.netty</groupId>
-            <artifactId>netty-common</artifactId>
-            <version>4.1.82.Final</version>
-        </dependency>
-        <dependency>
-            <groupId>commons-codec</groupId>
-            <artifactId>commons-codec</artifactId>
-            <version>1.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.arrow</groupId>
-            <artifactId>arrow-memory-core</artifactId>
-            <version>10.0.0</version>
-            <exclusions>
-              <exclusion>
-                <groupId>org.slf4j</groupId>
-                <artifactId>slf4j-api</artifactId>
-              </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.parquet</groupId>
-            <artifactId>parquet-column</artifactId>
-            <version>1.12.3</version>
-            <exclusions>
-              <exclusion>
-                  <groupId>org.slf4j</groupId>
-                  <artifactId>slf4j-api</artifactId>
-              </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-lang3</artifactId>
-            <version>3.12.0</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId> org.hamcrest</groupId>
-            <artifactId>hamcrest-core</artifactId>
-            <version>1.3</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>31.1-jre</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.woodstox</groupId>
-            <artifactId>woodstox-core</artifactId>
-            <version>5.3.0</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.codehaus.woodstox</groupId>
-                    <artifactId>stax2-api</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>commons-logging</groupId>
-            <artifactId>commons-logging</artifactId>
-            <version>1.2</version>
-        </dependency>
-        <dependency>
-            <groupId>commons-io</groupId>
-            <artifactId>commons-io</artifactId>
-            <version>2.8.0</version>
-        </dependency>
-    </dependencies>
+  <build>
+    <finalName>${project.artifactId}</finalName>
+    <resources>
+      <resource>
+        <filtering>true</filtering>
+        <directory>src/main/resources</directory>
+      </resource>
+    </resources>
+
+    <!-- disable default maven deploy plugin since we are using gpg:sign-and-deploy-file -->
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <artifactId>maven-deploy-plugin</artifactId>
+          <configuration>
+            <skip>true</skip>
+          </configuration>
+        </plugin>
+        <!-- Need to define the maven-failsafe-plugin here -->
+        <!-- http://maven.apache.org/surefire/maven-failsafe-plugin/plugin-info.html -->
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-failsafe-plugin</artifactId>
+          <version>3.0.0-M5</version>
+        </plugin>
+        <plugin>
+          <groupId>org.jacoco</groupId>
+          <artifactId>jacoco-maven-plugin</artifactId>
+          <version>${jacoco.version}</version>
+          <configuration>
+            <skip>${jacoco.skip.instrument}</skip>
+          </configuration>
+          <executions>
+            <execution>
+              <id>pre-unit-test</id>
+              <goals>
+                <goal>prepare-agent</goal>
+              </goals>
+              <configuration>
+                <destFile>target/jacoco-ut.exec</destFile>
+              </configuration>
+            </execution>
+            <execution>
+              <id>post-unit-test</id>
+              <goals>
+                <goal>report</goal>
+              </goals>
+              <phase>test</phase>
+              <configuration>
+                <dataFile>target/jacoco-ut.exec</dataFile>
+                <outputDirectory>target/jacoco-ut</outputDirectory>
+              </configuration>
+            </execution>
+          </executions>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+    <plugins>
+      <plugin>
+        <groupId>com.github.ekryd.sortpom</groupId>
+        <artifactId>sortpom-maven-plugin</artifactId>
+        <version>3.0.1</version>
+        <configuration>
+          <createBackupFile>false</createBackupFile>
+          <expandEmptyElements>false</expandEmptyElements>
+          <indentSchemaLocation>true</indentSchemaLocation>
+          <sortDependencies>scope,groupId,artifactId</sortDependencies>
+          <sortDependencyExclusions>groupId,artifactId</sortDependencyExclusions>
+          <sortExecutions>true</sortExecutions>
+          <sortModules>true</sortModules>
+          <sortPlugins>groupId,artifactId</sortPlugins>
+          <sortProperties>true</sortProperties>
+          <verifyFail>stop</verifyFail>
+          <verifyFailOn>strict</verifyFailOn>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>verify</goal>
+            </goals>
+            <phase>validate</phase>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <configuration>
+          <archive>
+            <manifest>
+              <addClasspath>true</addClasspath>
+              <mainClass>net.snowflake.ingest.SimpleIngestManager</mainClass>
+            </manifest>
+          </archive>
+          <descriptorRefs>
+            <descriptorRef>jar-with-dependencies</descriptorRef>
+          </descriptorRefs>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>2.5.1</version>
+        <inherited>true</inherited>
+        <configuration>
+          <source>1.8</source>
+          <target>1.8</target>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <version>3.3.0</version>
+        <executions>
+          <execution>
+            <id>analyze</id>
+            <goals>
+              <goal>analyze-only</goal>
+            </goals>
+            <configuration>
+              <failOnWarning>true</failOnWarning>
+              <ignoreNonCompile>true</ignoreNonCompile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <version>2.10.4</version>
+        <executions>
+          <execution>
+            <id>attach-javadocs</id>
+            <goals>
+              <goal>jar</goal>
+            </goals>
+            <configuration>
+              <additionalparam>-Xdoclint:none</additionalparam>
+              <source>8</source>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-source-plugin</artifactId>
+        <version>3.0.1</version>
+        <executions>
+          <execution>
+            <id>attach-sources</id>
+            <goals>
+              <goal>jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>2.19.1</version>
+        <configuration>
+          <excludes>
+            <exclude>**/TestSimpleIngestLocal.java</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>check-shaded-content</id>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <phase>verify</phase>
+            <configuration>
+              <executable>${basedir}/scripts/check_content.sh</executable>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
 
-    <profiles>
-        <profile>
-            <id>shadeDep</id>
-            <activation>
-                <property>
-                    <name>!not-shadeDep</name>
-                </property>
-            </activation>
-            <build>
-                <plugins>
-                    <!-- Relocate all dependencies to internal to solve dependency conflict problem -->
-                    <plugin>
-                        <groupId>org.apache.maven.plugins</groupId>
-                        <artifactId>maven-shade-plugin</artifactId>
-                        <version>3.1.1</version>
-                        <configuration>
-                            <relocations>
-                                <relocation>
-                                    <pattern>com.nimbusds</pattern>
-                                    <shadedPattern>
-                                        ${shadeBase}.com.nimbusds
-                                    </shadedPattern>
-                                </relocation>
-                                <relocation>
-                                    <pattern>net.jcip</pattern>
-                                    <shadedPattern>${shadeBase}.net.jcip</shadedPattern>
-                                </relocation>
-                                <relocation>
-                                    <pattern>net.minidev</pattern>
-                                    <shadedPattern>${shadeBase}.net.minidev</shadedPattern>
-                                </relocation>
-                                <relocation>
-                                    <pattern>org.objectweb</pattern>
-                                    <shadedPattern>${shadeBase}.org.objectweb</shadedPattern>
-                                </relocation>
-                                <relocation>
-                                    <pattern>com.fasterxml</pattern>
-                                    <shadedPattern>${shadeBase}.fasterxml</shadedPattern>
-                                </relocation>
-                                <relocation>
-                                    <pattern>org.apache</pattern>
-                                    <shadedPattern>${shadeBase}.apache</shadedPattern>
-                                </relocation>
-                            </relocations>
-                            <filters>
-                                <filter>
-                                    <artifact>*:*</artifact>
-                                    <excludes>
-                                        <exclude>META-INF/LICENSE*</exclude>
-                                        <exclude>META-INF/NOTICE*</exclude>
-                                        <exclude>META-INF/DEPENDENCIES</exclude>
-                                        <exclude>META-INF/maven/**</exclude>
-                                        <exclude>META-INF/services/com.fasterxml.*</exclude>
-                                        <exclude>META-INF/*.xml</exclude>
-                                        <exclude>META-INF/*.SF</exclude>
-                                        <exclude>META-INF/*.DSA</exclude>
-                                        <exclude>META-INF/*.RSA</exclude>
-                                    </excludes>
-                                </filter>
-                                <filter>
-                                    <artifact>commons-logging:commons-logging</artifact>
-                                    <excludes>
-                                        <exclude>org/apache/commons/logging/impl/AvalonLogger.class</exclude>
-                                    </excludes>
-                                </filter>
-                                <filter>
-                                    <artifact>org.slf4j:slf4j-simple</artifact>
-                                    <excludes>
-                                        <exclude>org/slf4j/**</exclude>
-                                    </excludes>
-                                </filter>
-                            </filters>
-                            <transformers>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                </transformer>
-                            </transformers>
-                        </configuration>
-                        <executions>
-                            <execution>
-                                <phase>package</phase>
-                                <goals>
-                                    <goal>shade</goal>
-                                </goals>
-                            </execution>
-                        </executions>
-                    </plugin>
-                </plugins>
-            </build>
-        </profile>
-        <profile>
-            <id>ossrh-deploy</id>
-            <activation>
-                <property>
-                    <name>ossrh-deploy</name>
-                </property>
-            </activation>
-            <build>
-                <plugins>
-                    <plugin>
-                        <groupId>org.apache.maven.plugins</groupId>
-                        <artifactId>maven-gpg-plugin</artifactId>
-                        <version>1.6</version>
-                        <executions>
-                            <execution>
-                                <phase>deploy</phase>
-                                <goals>
-                                    <goal>sign-and-deploy-file</goal>
-                                </goals>
-                                <configuration>
-                                    <file>target/${project.artifactId}.jar</file>
-                                    <repositoryId>ossrh</repositoryId>
-                                    <url>https://oss.sonatype.org/service/local/staging/deploy/maven2</url>
-                                    <pomFile>generated_public_pom.xml</pomFile>
-                                    <javadoc>target/${project.artifactId}-javadoc.jar</javadoc>
-                                    <sources>target/${project.artifactId}-sources.jar</sources>
-                                    <keyname>${env.GPG_KEY_ID}</keyname>
-                                    <passphrase>${env.GPG_KEY_PASSPHRASE}</passphrase>
-                                </configuration>
-                            </execution>
-                        </executions>
-                    </plugin>
-                </plugins>
-            </build>
-        </profile>
-        <profile>
-            <id>ghActionsIT</id>
-            <activation>
-                <property>
-                    <name>ghActionsIT</name>
-                </property>
-            </activation>
-            <build>
-                <plugins>
-                    <plugin>
-                        <groupId>org.apache.maven.plugins</groupId>
-                        <artifactId>maven-failsafe-plugin</artifactId>
-                        <executions>
-                            <execution>
-                                <goals>
-                                    <goal>integration-test</goal>
-                                </goals>
-                            </execution>
-                            <execution>
-                                <id>verify_github_actions_it</id>
-                                <phase>verify</phase>
-                                <goals>
-                                    <goal>verify</goal>
-                                </goals>
-                            </execution>
-                        </executions>
-                    </plugin>
-                    <plugin>
-                        <groupId>org.jacoco</groupId>
-                        <artifactId>jacoco-maven-plugin</artifactId>
-                        <version>${jacoco.version}</version>
-                        <executions>
-                            <execution>
-                                <id>pre-unit-test</id>
-                                <goals>
-                                    <goal>prepare-agent</goal>
-                                </goals>
-                                <configuration>
-                                    <destFile>target/jacoco-ut.exec</destFile>
-                                </configuration>
-                            </execution>
-                            <execution>
-                                <id>post-unit-test</id>
-                                <phase>test</phase>
-                                <goals>
-                                    <goal>report</goal>
-                                </goals>
-                                <configuration>
-                                    <dataFile>target/jacoco-ut.exec</dataFile>
-                                    <outputDirectory>target/jacoco-ut</outputDirectory>
-                                </configuration>
-                            </execution>
-                            <execution>
-                                <id>pre-integration-test</id>
-                                <phase>pre-integration-test</phase>
-                                <goals>
-                                    <goal>prepare-agent</goal>
-                                </goals>
-                                <configuration>
-                                    <destFile>target/jacoco-it.exec</destFile>
-                                </configuration>
-                            </execution>
-                            <execution>
-                                <id>post-integration-test</id>
-                                <phase>post-integration-test</phase>
-                                <goals>
-                                    <goal>report</goal>
-                                </goals>
-                                <configuration>
-                                    <dataFile>target/jacoco-it.exec</dataFile>
-                                    <outputDirectory>target/jacoco-it</outputDirectory>
-                                </configuration>
-                            </execution>
-                        </executions>
-                        <configuration>
-                            <skip>${jacoco.skip.instrument}</skip>
-                        </configuration>
-                    </plugin>
-                </plugins>
-            </build>
-        </profile>
-        <profile>
-          <id>run-enforcer</id>
-            <activation>
-              <activeByDefault>false</activeByDefault>
-            </activation>
-          <build>
-            <plugins>
-              <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-enforcer-plugin</artifactId>
-                <version>3.0.0-M3</version>
-                <dependencies>
-                  <dependency>
-                    <groupId>com.google.cloud.tools</groupId>
-                    <artifactId>linkage-checker-enforcer-rules</artifactId>
-                    <version>1.5.12</version>
-                  </dependency>
-                  <dependency>
-                    <groupId>org.codehaus.mojo</groupId>
-                    <artifactId>extra-enforcer-rules</artifactId>
-                    <version>1.3</version>
-                    <exclusions>
-                      <exclusion>
-                        <groupId>org.eclipse.aether</groupId>
-                        <artifactId>aether-util</artifactId>
-                      </exclusion>
-                    </exclusions>
-                  </dependency>
-                </dependencies>
-                <executions>
-                  <execution>
-                    <id>enforce-best-practices</id>
-                    <goals>
-                      <goal>enforce</goal>
-                    </goals>
-                    <configuration>
-                      <rules>
-                        <banDuplicateClasses>
-                          <findAllDuplicates>true</findAllDuplicates>
-                          <ignoreWhenIdentical>true</ignoreWhenIdentical>
+  <profiles>
+    <profile>
+      <id>shadeDep</id>
+      <activation>
+        <property>
+          <name>!not-shadeDep</name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <!-- Relocate all dependencies to internal to solve dependency conflict problem -->
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-shade-plugin</artifactId>
+            <version>3.1.1</version>
+            <configuration>
+              <relocations>
+                <relocation>
+                  <pattern>com.nimbusds</pattern>
+                  <shadedPattern>${shadeBase}.com.nimbusds</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>net.jcip</pattern>
+                  <shadedPattern>${shadeBase}.net.jcip</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>net.minidev</pattern>
+                  <shadedPattern>${shadeBase}.net.minidev</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.objectweb</pattern>
+                  <shadedPattern>${shadeBase}.org.objectweb</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>com.fasterxml</pattern>
+                  <shadedPattern>${shadeBase}.fasterxml</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache</pattern>
+                  <shadedPattern>${shadeBase}.apache</shadedPattern>
+                </relocation>
+              </relocations>
+              <filters>
+                <filter>
+                  <artifact>*:*</artifact>
+                  <excludes>
+                    <exclude>META-INF/LICENSE*</exclude>
+                    <exclude>META-INF/NOTICE*</exclude>
+                    <exclude>META-INF/DEPENDENCIES</exclude>
+                    <exclude>META-INF/maven/**</exclude>
+                    <exclude>META-INF/services/com.fasterxml.*</exclude>
+                    <exclude>META-INF/*.xml</exclude>
+                    <exclude>META-INF/*.SF</exclude>
+                    <exclude>META-INF/*.DSA</exclude>
+                    <exclude>META-INF/*.RSA</exclude>
+                  </excludes>
+                </filter>
+                <filter>
+                  <artifact>commons-logging:commons-logging</artifact>
+                  <excludes>
+                    <exclude>org/apache/commons/logging/impl/AvalonLogger.class</exclude>
+                  </excludes>
+                </filter>
+                <filter>
+                  <artifact>org.slf4j:slf4j-simple</artifact>
+                  <excludes>
+                    <exclude>org/slf4j/**</exclude>
+                  </excludes>
+                </filter>
+              </filters>
+              <transformers>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"/>
+              </transformers>
+            </configuration>
+            <executions>
+              <execution>
+                <goals>
+                  <goal>shade</goal>
+                </goals>
+                <phase>package</phase>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <id>ossrh-deploy</id>
+      <activation>
+        <property>
+          <name>ossrh-deploy</name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-gpg-plugin</artifactId>
+            <version>1.6</version>
+            <executions>
+              <execution>
+                <goals>
+                  <goal>sign-and-deploy-file</goal>
+                </goals>
+                <phase>deploy</phase>
+                <configuration>
+                  <file>target/${project.artifactId}.jar</file>
+                  <repositoryId>ossrh</repositoryId>
+                  <url>https://oss.sonatype.org/service/local/staging/deploy/maven2</url>
+                  <pomFile>generated_public_pom.xml</pomFile>
+                  <javadoc>target/${project.artifactId}-javadoc.jar</javadoc>
+                  <sources>target/${project.artifactId}-sources.jar</sources>
+                  <keyname>${env.GPG_KEY_ID}</keyname>
+                  <passphrase>${env.GPG_KEY_PASSPHRASE}</passphrase>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <id>ghActionsIT</id>
+      <activation>
+        <property>
+          <name>ghActionsIT</name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-failsafe-plugin</artifactId>
+            <executions>
+              <execution>
+                <goals>
+                  <goal>integration-test</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>verify_github_actions_it</id>
+                <goals>
+                  <goal>verify</goal>
+                </goals>
+                <phase>verify</phase>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.jacoco</groupId>
+            <artifactId>jacoco-maven-plugin</artifactId>
+            <version>${jacoco.version}</version>
+            <configuration>
+              <skip>${jacoco.skip.instrument}</skip>
+            </configuration>
+            <executions>
+              <execution>
+                <id>pre-unit-test</id>
+                <goals>
+                  <goal>prepare-agent</goal>
+                </goals>
+                <configuration>
+                  <destFile>target/jacoco-ut.exec</destFile>
+                </configuration>
+              </execution>
+              <execution>
+                <id>post-unit-test</id>
+                <goals>
+                  <goal>report</goal>
+                </goals>
+                <phase>test</phase>
+                <configuration>
+                  <dataFile>target/jacoco-ut.exec</dataFile>
+                  <outputDirectory>target/jacoco-ut</outputDirectory>
+                </configuration>
+              </execution>
+              <execution>
+                <id>pre-integration-test</id>
+                <goals>
+                  <goal>prepare-agent</goal>
+                </goals>
+                <phase>pre-integration-test</phase>
+                <configuration>
+                  <destFile>target/jacoco-it.exec</destFile>
+                </configuration>
+              </execution>
+              <execution>
+                <id>post-integration-test</id>
+                <goals>
+                  <goal>report</goal>
+                </goals>
+                <phase>post-integration-test</phase>
+                <configuration>
+                  <dataFile>target/jacoco-it.exec</dataFile>
+                  <outputDirectory>target/jacoco-it</outputDirectory>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <id>run-enforcer</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-enforcer-plugin</artifactId>
+            <version>3.0.0-M3</version>
+            <dependencies>
+              <dependency>
+                <groupId>com.google.cloud.tools</groupId>
+                <artifactId>linkage-checker-enforcer-rules</artifactId>
+                <version>1.5.12</version>
+              </dependency>
+              <dependency>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>extra-enforcer-rules</artifactId>
+                <version>1.3</version>
+                <exclusions>
+                  <exclusion>
+                    <groupId>org.eclipse.aether</groupId>
+                    <artifactId>aether-util</artifactId>
+                  </exclusion>
+                </exclusions>
+              </dependency>
+            </dependencies>
+            <executions>
+              <execution>
+                <id>enforce-best-practices</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <configuration>
+                  <rules>
+                    <banDuplicateClasses>
+                      <dependencies>
+                        <dependency>
+                          <groupId>log4j</groupId>
+                          <artifactId>log4j</artifactId>
                           <ignoreClasses>
-                            <!-- <ignoreClass>com.snowflake.jmh.*</ignoreClass> -->
+                            <ignoreClass>*</ignoreClass>
                           </ignoreClasses>
-                        </banDuplicateClasses>
-                        <banDuplicatePomDependencyVersions/>
-                        <bannedDependencies>
-                          <excludes>
-                            <!-- <exclude>org.bouncycastle:bcprov-jdk15on</exclude> -->
-                          </excludes>
-                        </bannedDependencies>
-                        <dependencyConvergence/>
-                        <requireUpperBoundDeps/>
-                      </rules>
-                    </configuration>
-                  </execution>
-                  <execution>
-                    <id>enforce-maven</id>
-                    <goals>
-                      <goal>enforce</goal>
-                    </goals>
-                    <configuration>
-                      <rules>
-                        <requireMavenVersion>
-                        <version>3.5</version>
-                        </requireMavenVersion>
-                      </rules>
-                    </configuration>
-                  </execution>
-                  <execution>
-                    <id>enforce-linkage-checker</id>
-                    <goals>
-                      <goal>enforce</goal>
-                    </goals>
-                    <phase>verify</phase>
-                    <configuration>
-                      <rules>
-                        <LinkageCheckerRule implementation="com.google.cloud.tools.dependencies.enforcer.LinkageCheckerRule">
-                          <reportOnlyReachable>true</reportOnlyReachable>
-                          <!-- <exclusionFile>${basedir}/linkage-checker-exclusion-rules.xml</exclusionFile> -->
-                        </LinkageCheckerRule>
-                        <bannedDependencies>
-                          <excludes>
-                            <!-- JUnit 4 is not allowed on the classpath to prevent its accidental use -->
-                            <!-- <exclude>junit:junit</exclude> -->
-                          </excludes>
-                        </bannedDependencies>
-                      </rules>
-                    </configuration>
-                  </execution>
-                </executions>
-              </plugin>
-            </plugins>
-          </build>
-        </profile>
-    </profiles>
+                        </dependency>
+                        <dependency>
+                          <groupId>javax.activation</groupId>
+                          <artifactId>activation</artifactId>
+                          <ignoreClasses>
+                            <ignoreClass>*</ignoreClass>
+                          </ignoreClasses>
+                        </dependency>
+                        <dependency>
+                          <groupId>org.slf4j</groupId>
+                          <artifactId>slf4j-simple</artifactId>
+                          <ignoreClasses>
+                            <ignoreClass>*</ignoreClass>
+                          </ignoreClasses>
+                        </dependency>
+                        <dependency>
+                          <groupId>org.slf4j</groupId>
+                          <artifactId>slf4j-reload4j</artifactId>
+                          <ignoreClasses>
+                            <ignoreClass>*</ignoreClass>
+                          </ignoreClasses>
+                        </dependency>
+                      </dependencies>
+                      <findAllDuplicates>true</findAllDuplicates>
+                      <ignoreWhenIdentical>true</ignoreWhenIdentical>
+                    </banDuplicateClasses>
+                    <banDuplicatePomDependencyVersions/>
+                    <bannedDependencies>
+                      <excludes>
+                        <!-- <exclude>org.bouncycastle:bcprov-jdk15on</exclude> -->
+                      </excludes>
+                    </bannedDependencies>
+                    <dependencyConvergence/>
+                    <requireUpperBoundDeps/>
+                  </rules>
+                </configuration>
+              </execution>
+              <execution>
+                <id>enforce-maven</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <configuration>
+                  <rules>
+                    <requireMavenVersion>
+                      <version>3.5</version>
+                    </requireMavenVersion>
+                  </rules>
+                </configuration>
+              </execution>
+              <execution>
+                <id>enforce-linkage-checker</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <phase>verify</phase>
+                <configuration>
+                  <rules>
+                    <LinkageCheckerRule implementation="com.google.cloud.tools.dependencies.enforcer.LinkageCheckerRule">
+                      <reportOnlyReachable>true</reportOnlyReachable>
+                      <exclusionFile>${basedir}/linkage-checker-exclusion-rules.xml</exclusionFile>
+                      <level>WARN</level>
+                    </LinkageCheckerRule>
+                    <bannedDependencies>
+                      <excludes>
+                        <!-- JUnit 4 is not allowed on the classpath to prevent its accidental use -->
+                        <!-- <exclude>junit:junit</exclude> -->
+                      </excludes>
+                    </bannedDependencies>
+                  </rules>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
 
 </project>

From 782fa4a7e33714872b6da48c620f28d64e790ed0 Mon Sep 17 00:00:00 2001
From: David Wang <david.wang@snowflake.com>
Date: Wed, 22 Mar 2023 20:55:56 +0000
Subject: [PATCH 142/356] SNOW-765525 address comment

---
 pom.xml | 234 +++++++++++++++++++++++++++-----------------------------
 1 file changed, 112 insertions(+), 122 deletions(-)

diff --git a/pom.xml b/pom.xml
index 2ba6bf1aa..6138578af 100644
--- a/pom.xml
+++ b/pom.xml
@@ -471,6 +471,118 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <version>3.0.0-M3</version>
+        <dependencies>
+          <dependency>
+            <groupId>com.google.cloud.tools</groupId>
+            <artifactId>linkage-checker-enforcer-rules</artifactId>
+            <version>1.5.12</version>
+          </dependency>
+          <dependency>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>extra-enforcer-rules</artifactId>
+            <version>1.3</version>
+            <exclusions>
+              <exclusion>
+                <groupId>org.eclipse.aether</groupId>
+                <artifactId>aether-util</artifactId>
+              </exclusion>
+            </exclusions>
+          </dependency>
+        </dependencies>
+        <executions>
+          <execution>
+            <id>enforce-best-practices</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <configuration>
+              <rules>
+                <banDuplicateClasses>
+                  <dependencies>
+                    <dependency>
+                      <groupId>log4j</groupId>
+                      <artifactId>log4j</artifactId>
+                      <ignoreClasses>
+                        <ignoreClass>*</ignoreClass>
+                      </ignoreClasses>
+                    </dependency>
+                    <dependency>
+                      <groupId>javax.activation</groupId>
+                      <artifactId>activation</artifactId>
+                      <ignoreClasses>
+                        <ignoreClass>*</ignoreClass>
+                      </ignoreClasses>
+                    </dependency>
+                    <dependency>
+                      <groupId>org.slf4j</groupId>
+                      <artifactId>slf4j-simple</artifactId>
+                      <ignoreClasses>
+                        <ignoreClass>*</ignoreClass>
+                      </ignoreClasses>
+                    </dependency>
+                    <dependency>
+                      <groupId>org.slf4j</groupId>
+                      <artifactId>slf4j-reload4j</artifactId>
+                      <ignoreClasses>
+                        <ignoreClass>*</ignoreClass>
+                      </ignoreClasses>
+                    </dependency>
+                  </dependencies>
+                  <findAllDuplicates>true</findAllDuplicates>
+                  <ignoreWhenIdentical>true</ignoreWhenIdentical>
+                </banDuplicateClasses>
+                <banDuplicatePomDependencyVersions/>
+                <bannedDependencies>
+                  <excludes>
+                    <!-- <exclude>org.bouncycastle:bcprov-jdk15on</exclude> -->
+                  </excludes>
+                </bannedDependencies>
+                <dependencyConvergence/>
+                <requireUpperBoundDeps/>
+              </rules>
+            </configuration>
+          </execution>
+          <execution>
+            <id>enforce-maven</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <configuration>
+              <rules>
+                <requireMavenVersion>
+                  <version>3.5</version>
+                </requireMavenVersion>
+              </rules>
+            </configuration>
+          </execution>
+          <execution>
+            <id>enforce-linkage-checker</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <phase>verify</phase>
+            <configuration>
+              <rules>
+                <LinkageCheckerRule implementation="com.google.cloud.tools.dependencies.enforcer.LinkageCheckerRule">
+                  <reportOnlyReachable>true</reportOnlyReachable>
+                  <exclusionFile>${basedir}/linkage-checker-exclusion-rules.xml</exclusionFile>
+                  <level>WARN</level>
+                </LinkageCheckerRule>
+                <bannedDependencies>
+                  <excludes>
+                    <!-- JUnit 4 is not allowed on the classpath to prevent its accidental use -->
+                    <!-- <exclude>junit:junit</exclude> -->
+                  </excludes>
+                </bannedDependencies>
+              </rules>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-javadoc-plugin</artifactId>
@@ -733,128 +845,6 @@
         </plugins>
       </build>
     </profile>
-    <profile>
-      <id>run-enforcer</id>
-      <activation>
-        <activeByDefault>false</activeByDefault>
-      </activation>
-      <build>
-        <plugins>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-enforcer-plugin</artifactId>
-            <version>3.0.0-M3</version>
-            <dependencies>
-              <dependency>
-                <groupId>com.google.cloud.tools</groupId>
-                <artifactId>linkage-checker-enforcer-rules</artifactId>
-                <version>1.5.12</version>
-              </dependency>
-              <dependency>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>extra-enforcer-rules</artifactId>
-                <version>1.3</version>
-                <exclusions>
-                  <exclusion>
-                    <groupId>org.eclipse.aether</groupId>
-                    <artifactId>aether-util</artifactId>
-                  </exclusion>
-                </exclusions>
-              </dependency>
-            </dependencies>
-            <executions>
-              <execution>
-                <id>enforce-best-practices</id>
-                <goals>
-                  <goal>enforce</goal>
-                </goals>
-                <configuration>
-                  <rules>
-                    <banDuplicateClasses>
-                      <dependencies>
-                        <dependency>
-                          <groupId>log4j</groupId>
-                          <artifactId>log4j</artifactId>
-                          <ignoreClasses>
-                            <ignoreClass>*</ignoreClass>
-                          </ignoreClasses>
-                        </dependency>
-                        <dependency>
-                          <groupId>javax.activation</groupId>
-                          <artifactId>activation</artifactId>
-                          <ignoreClasses>
-                            <ignoreClass>*</ignoreClass>
-                          </ignoreClasses>
-                        </dependency>
-                        <dependency>
-                          <groupId>org.slf4j</groupId>
-                          <artifactId>slf4j-simple</artifactId>
-                          <ignoreClasses>
-                            <ignoreClass>*</ignoreClass>
-                          </ignoreClasses>
-                        </dependency>
-                        <dependency>
-                          <groupId>org.slf4j</groupId>
-                          <artifactId>slf4j-reload4j</artifactId>
-                          <ignoreClasses>
-                            <ignoreClass>*</ignoreClass>
-                          </ignoreClasses>
-                        </dependency>
-                      </dependencies>
-                      <findAllDuplicates>true</findAllDuplicates>
-                      <ignoreWhenIdentical>true</ignoreWhenIdentical>
-                    </banDuplicateClasses>
-                    <banDuplicatePomDependencyVersions/>
-                    <bannedDependencies>
-                      <excludes>
-                        <!-- <exclude>org.bouncycastle:bcprov-jdk15on</exclude> -->
-                      </excludes>
-                    </bannedDependencies>
-                    <dependencyConvergence/>
-                    <requireUpperBoundDeps/>
-                  </rules>
-                </configuration>
-              </execution>
-              <execution>
-                <id>enforce-maven</id>
-                <goals>
-                  <goal>enforce</goal>
-                </goals>
-                <configuration>
-                  <rules>
-                    <requireMavenVersion>
-                      <version>3.5</version>
-                    </requireMavenVersion>
-                  </rules>
-                </configuration>
-              </execution>
-              <execution>
-                <id>enforce-linkage-checker</id>
-                <goals>
-                  <goal>enforce</goal>
-                </goals>
-                <phase>verify</phase>
-                <configuration>
-                  <rules>
-                    <LinkageCheckerRule implementation="com.google.cloud.tools.dependencies.enforcer.LinkageCheckerRule">
-                      <reportOnlyReachable>true</reportOnlyReachable>
-                      <exclusionFile>${basedir}/linkage-checker-exclusion-rules.xml</exclusionFile>
-                      <level>WARN</level>
-                    </LinkageCheckerRule>
-                    <bannedDependencies>
-                      <excludes>
-                        <!-- JUnit 4 is not allowed on the classpath to prevent its accidental use -->
-                        <!-- <exclude>junit:junit</exclude> -->
-                      </excludes>
-                    </bannedDependencies>
-                  </rules>
-                </configuration>
-              </execution>
-            </executions>
-          </plugin>
-        </plugins>
-      </build>
-    </profile>
   </profiles>
 
 </project>

From d072c8c600bb7c7ae46b37fb3905404f883ea0be Mon Sep 17 00:00:00 2001
From: David Wang <david.wang@snowflake.com>
Date: Thu, 23 Mar 2023 07:19:11 +0000
Subject: [PATCH 143/356] SNOW-765525 fix all dependency linkage issue and more
 cleanup

---
 linkage-checker-exclusion-rules.xml |  30 ++++-
 pom.xml                             | 181 ++++++++++++++--------------
 2 files changed, 118 insertions(+), 93 deletions(-)

diff --git a/linkage-checker-exclusion-rules.xml b/linkage-checker-exclusion-rules.xml
index 66e1455c3..ecb2afe9d 100644
--- a/linkage-checker-exclusion-rules.xml
+++ b/linkage-checker-exclusion-rules.xml
@@ -1,8 +1,28 @@
 <LinkageCheckerFilter>
-  <!-- <LinkageError>
-           <Source><Package name=""/></Source>
-    <Target><Package name=""/></Target>
-    <Reason></Reason>
-  </LinkageError> -->
+  <LinkageError>
+    <Source><Package name="com.nimbusds.jose"/></Source>
+    <Reason>Optional</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Package name="org.conscrypt"/></Target>
+    <Reason>Optional</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Package name="com.sun.jna"/></Target>
+    <Reason>Optional</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Package name="org.tukaani.xz"/></Target>
+    <Reason>Optional</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="org.apache.hadoop.mapred"/></Source>
+    <Reason>Conflict due to parquet-hadoop uses provided scope of hadoo-common</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.client.jdbc.internal"/></Source>
+    <Reason>Unknown: Likely the referenced class is not used in the jdbc package</Reason>
+  </LinkageError>
+
 </LinkageCheckerFilter>
 
diff --git a/pom.xml b/pom.xml
index 6138578af..d993795c9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -36,101 +36,157 @@
   <!-- Set our Language Level to Java 8 -->
   <properties>
     <arrow.version>10.0.0</arrow.version>
+    <codehaus.version>1.9.13</codehaus.version>
+    <commonscodec.version>1.15</commonscodec.version>
+    <commonscollections.version>3.2.2</commonscollections.version>
+    <commonscompress.version>1.21</commonscompress.version>
+    <commonsio.version>2.8.0</commonsio.version>
+    <commonslang3.version>3.12.0</commonslang3.version>
+    <commonslogging.version>1.2</commonslogging.version>
+    <fasterxml.version>2.14.0</fasterxml.version>
+    <guava.version>31.1-jre</guava.version>
     <hadoop.version>3.3.4</hadoop.version>
     <jacoco.skip.instrument>true</jacoco.skip.instrument>
     <jacoco.version>0.8.5</jacoco.version>
     <maven.compiler.source>1.8</maven.compiler.source>
     <maven.compiler.target>1.8</maven.compiler.target>
+    <netty.version>4.1.82.Final</netty.version>
+    <nimbusds.version>9.9.3</nimbusds.version>
+    <objenesis.version>3.1</objenesis.version>
+    <parquet.version>1.12.3</parquet.version>
     <shadeBase>net.snowflake.ingest.internal</shadeBase>
+    <slf4j.version>1.7.36</slf4j.version>
+    <snappy.version>1.1.8.3</snappy.version>
+    <snowjdbc.version>3.13.15</snowjdbc.version>
+    <yetus.version>0.13.0</yetus.version>
   </properties>
 
   <dependencyManagement>
     <dependencies>
       <dependency>
-        <groupId>ch.qos.reload4j</groupId>
-        <artifactId>reload4j</artifactId>
-        <version>1.2.22</version>
-      </dependency>
-      <dependency>
-        <groupId>com.fasterxml.jackson.core</groupId>
-        <artifactId>jackson-annotations</artifactId>
-        <version>2.14.0</version>
-      </dependency>
-      <dependency>
-        <groupId>com.fasterxml.jackson.core</groupId>
-        <artifactId>jackson-core</artifactId>
-        <version>2.14.0</version>
-      </dependency>
-      <!-- Jackson Databind api -->
-      <dependency>
-        <groupId>com.fasterxml.jackson.core</groupId>
-        <artifactId>jackson-databind</artifactId>
-        <version>2.14.0</version>
+        <groupId>com.fasterxml.jackson</groupId>
+        <artifactId>jackson-bom</artifactId>
+        <version>${fasterxml.version}</version>
+        <type>pom</type>
+        <scope>import</scope>
       </dependency>
       <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava</artifactId>
-        <version>31.1-jre</version>
+        <version>${guava.version}</version>
       </dependency>
       <dependency>
         <groupId>com.nimbusds</groupId>
         <artifactId>nimbus-jose-jwt</artifactId>
-        <version>9.9.3</version>
+        <version>${nimbusds.version}</version>
       </dependency>
       <dependency>
         <groupId>commons-codec</groupId>
         <artifactId>commons-codec</artifactId>
-        <version>1.15</version>
+        <version>${commonscodec.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>commons-collections</groupId>
+        <artifactId>commons-collections</artifactId>
+        <version>${commonscollections.version}</version>
       </dependency>
       <dependency>
         <groupId>commons-io</groupId>
         <artifactId>commons-io</artifactId>
-        <version>2.8.0</version>
+        <version>${commonsio.version}</version>
       </dependency>
       <dependency>
         <groupId>commons-logging</groupId>
         <artifactId>commons-logging</artifactId>
-        <version>1.2</version>
+        <version>${commonslogging.version}</version>
       </dependency>
       <dependency>
         <groupId>io.netty</groupId>
         <artifactId>netty-buffer</artifactId>
-        <version>4.1.82.Final</version>
+        <version>${netty.version}</version>
       </dependency>
       <dependency>
         <groupId>io.netty</groupId>
         <artifactId>netty-common</artifactId>
-        <version>4.1.82.Final</version>
+        <version>${netty.version}</version>
       </dependency>
       <dependency>
         <groupId>net.snowflake</groupId>
         <artifactId>snowflake-jdbc</artifactId>
-        <version>3.13.15</version>
+        <version>${snowjdbc.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.commons</groupId>
         <artifactId>commons-compress</artifactId>
-        <version>1.21</version>
+        <version>${commonscompress.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.commons</groupId>
         <artifactId>commons-lang3</artifactId>
-        <version>3.12.0</version>
+        <version>${commonslang3.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-common</artifactId>
+        <version>${hadoop.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>ch.qos.reload4j</groupId>
+            <artifactId>reload4j</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>javax.activation</groupId>
+            <artifactId>activation</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-reload4j</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.parquet</groupId>
+        <artifactId>parquet-column</artifactId>
+        <version>${parquet.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.parquet</groupId>
+        <artifactId>parquet-common</artifactId>
+        <version>${parquet.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.parquet</groupId>
+        <artifactId>parquet-hadoop</artifactId>
+        <version>${parquet.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.yetus</groupId>
         <artifactId>audience-annotations</artifactId>
-        <version>0.13.0</version>
+        <version>${yetus.version}</version>
       </dependency>
       <dependency>
         <groupId>org.codehaus.jackson</groupId>
         <artifactId>jackson-core-asl</artifactId>
-        <version>1.9.13</version>
+        <version>${codehaus.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.codehaus.jackson</groupId>
+        <artifactId>jackson-jaxrs</artifactId>
+        <version>${codehaus.version}</version>
       </dependency>
       <dependency>
         <groupId>org.codehaus.jackson</groupId>
         <artifactId>jackson-mapper-asl</artifactId>
-        <version>1.9.13</version>
+        <version>${codehaus.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.codehaus.jackson</groupId>
+        <artifactId>jackson-xc</artifactId>
+        <version>${codehaus.version}</version>
       </dependency>
       <dependency>
         <groupId>org.codehaus.woodstox</groupId>
@@ -140,24 +196,18 @@
       <dependency>
         <groupId>org.objenesis</groupId>
         <artifactId>objenesis</artifactId>
-        <version>3.1</version>
+        <version>${objenesis.version}</version>
       </dependency>
       <!-- SLF4J api that a client can shim in later -->
       <dependency>
         <groupId>org.slf4j</groupId>
         <artifactId>slf4j-api</artifactId>
-        <version>1.7.36</version>
-      </dependency>
-      <!-- JDK logger backend for logging tests -->
-      <dependency>
-        <groupId>org.slf4j</groupId>
-        <artifactId>slf4j-simple</artifactId>
-        <version>1.7.36</version>
+        <version>${slf4j.version}</version>
       </dependency>
       <dependency>
         <groupId>org.xerial.snappy</groupId>
         <artifactId>snappy-java</artifactId>
-        <version>1.1.8.3</version>
+        <version>${snappy.version}</version>
       </dependency>
 
       <!-- JUnit so that we can make some basic unit tests -->
@@ -252,58 +302,43 @@
       <groupId>net.bytebuddy</groupId>
       <artifactId>byte-buddy-agent</artifactId>
     </dependency>
-
     <!-- Snowflake JDBC used to connect to the service-->
     <dependency>
       <groupId>net.snowflake</groupId>
       <artifactId>snowflake-jdbc</artifactId>
     </dependency>
+    <!-- Apache Arrow -->
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-memory-core</artifactId>
       <version>${arrow.version}</version>
     </dependency>
-
-    <!-- Apache Arrow -->
     <dependency>
       <groupId>org.apache.arrow</groupId>
       <artifactId>arrow-vector</artifactId>
       <version>${arrow.version}</version>
     </dependency>
-
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
-      <version>${hadoop.version}</version>
     </dependency>
+    <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop -->
     <dependency>
       <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-column</artifactId>
-      <version>1.12.3</version>
     </dependency>
     <dependency>
       <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-common</artifactId>
-      <version>1.12.3</version>
     </dependency>
-
-    <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop -->
     <dependency>
       <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-hadoop</artifactId>
-      <version>1.12.3</version>
     </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.arrow</groupId>
-      <artifactId>arrow-memory-netty</artifactId>
-      <version>${arrow.version}</version>
-      <scope>runtime</scope>
-    </dependency>
-
     <!-- JUnit so that we can make some basic unit tests -->
     <dependency>
       <groupId>junit</groupId>
@@ -502,36 +537,6 @@
             <configuration>
               <rules>
                 <banDuplicateClasses>
-                  <dependencies>
-                    <dependency>
-                      <groupId>log4j</groupId>
-                      <artifactId>log4j</artifactId>
-                      <ignoreClasses>
-                        <ignoreClass>*</ignoreClass>
-                      </ignoreClasses>
-                    </dependency>
-                    <dependency>
-                      <groupId>javax.activation</groupId>
-                      <artifactId>activation</artifactId>
-                      <ignoreClasses>
-                        <ignoreClass>*</ignoreClass>
-                      </ignoreClasses>
-                    </dependency>
-                    <dependency>
-                      <groupId>org.slf4j</groupId>
-                      <artifactId>slf4j-simple</artifactId>
-                      <ignoreClasses>
-                        <ignoreClass>*</ignoreClass>
-                      </ignoreClasses>
-                    </dependency>
-                    <dependency>
-                      <groupId>org.slf4j</groupId>
-                      <artifactId>slf4j-reload4j</artifactId>
-                      <ignoreClasses>
-                        <ignoreClass>*</ignoreClass>
-                      </ignoreClasses>
-                    </dependency>
-                  </dependencies>
                   <findAllDuplicates>true</findAllDuplicates>
                   <ignoreWhenIdentical>true</ignoreWhenIdentical>
                 </banDuplicateClasses>

From 45c363e1698ea04eecdf86ff0f67723c7e068af8 Mon Sep 17 00:00:00 2001
From: David Wang <david.wang@snowflake.com>
Date: Thu, 23 Mar 2023 07:50:53 +0000
Subject: [PATCH 144/356] SNOW-765525 add back arrow netty runtime

---
 pom.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/pom.xml b/pom.xml
index d993795c9..2ae0cca98 100644
--- a/pom.xml
+++ b/pom.xml
@@ -339,6 +339,12 @@
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-memory-netty</artifactId>
+      <version>${arrow.version}</version>
+      <scope>runtime</scope>
+    </dependency>
     <!-- JUnit so that we can make some basic unit tests -->
     <dependency>
       <groupId>junit</groupId>

From e30632823f219ae4bd1300cf6fa540fdca9a614f Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Fri, 24 Mar 2023 13:51:43 +0100
Subject: [PATCH 145/356] SNOW-766522 Cache unquoted column names (#362)

SNOW-766522 Cache unquoted column names

The SDK is unnecessarily unquoting column names from user input again and again,
even though they are (likely) always the same in each row. This PR introduces a
guava loading cache of already computed unquoted column names to avoid unnecessary
re-computations.

A future PR will switch the cache implementation to scaffeine, which provides better performance.
---
 .../streaming/internal/LiteralQuoteUtils.java | 43 ++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/LiteralQuoteUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/LiteralQuoteUtils.java
index 285a62315..74d97cfa0 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/LiteralQuoteUtils.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/LiteralQuoteUtils.java
@@ -3,6 +3,13 @@
  */
 package net.snowflake.ingest.streaming.internal;
 
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import java.util.concurrent.ExecutionException;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.SFException;
+
 /**
  * Util class to normalise literals to match server side metadata.
  *
@@ -11,6 +18,40 @@
  */
 class LiteralQuoteUtils {
 
+  /** Maximum number of unquoted column names to store in cache */
+  static final int UNQUOTED_COLUMN_NAME_CACHE_MAX_SIZE = 30000;
+
+  /** Cache storing unquoted column names */
+  private static final LoadingCache<String, String> unquotedColumnNamesCache;
+
+  static {
+    unquotedColumnNamesCache =
+        CacheBuilder.newBuilder()
+            .maximumSize(UNQUOTED_COLUMN_NAME_CACHE_MAX_SIZE)
+            .build(
+                new CacheLoader<String, String>() {
+                  @Override
+                  public String load(String key) {
+                    return unquoteColumnNameInternal(key);
+                  }
+                });
+  }
+
+  /**
+   * Unquote column name expected to be used from the outside. It decides is unquoting would be
+   * expensive. If not, it unquotes directly, otherwise it return a value from a loading cache.
+   */
+  static String unquoteColumnName(String columnName) {
+    try {
+      return unquotedColumnNamesCache.get(columnName);
+    } catch (ExecutionException e) {
+      throw new SFException(
+          e,
+          ErrorCode.INTERNAL_ERROR,
+          String.format("Exception thrown while unquoting column name %s", columnName));
+    }
+  }
+
   /**
    * Unquote SQL literal.
    *
@@ -21,7 +62,7 @@ class LiteralQuoteUtils {
    * @param columnName column name literal to unquote
    * @return unquoted literal
    */
-  static String unquoteColumnName(String columnName) {
+  private static String unquoteColumnNameInternal(String columnName) {
     int length = columnName.length();
 
     if (length == 0) {

From 2130826a619f8ba36fa0090710a74f7903276ad1 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Fri, 24 Mar 2023 16:01:46 +0100
Subject: [PATCH 146/356] SNOW-735517 Fix nullCount/rowCount mismatch (#357)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

In `addRow`, the SDK iterates over columns of the row one-by-one, it
validates, calculates statistics and puts the resulting value in the
row buffer. The problem is when an exception is thrown (e.g. validation
fails), it (correctly) doesn’t flush the row, but it forgets to “undo”
the statistics calculated for the previous, already processed, columns
in the row.

This PR forks for each column during row processing and only updates
the resulting statistics when all column are processed successfully.
---
 .../streaming/internal/AbstractRowBuffer.java |  4 +-
 .../streaming/internal/ArrowRowBuffer.java    | 95 ++++++++++++-------
 .../streaming/internal/ParquetRowBuffer.java  | 24 ++++-
 .../streaming/internal/RowBufferStats.java    |  5 +
 ...owflakeStreamingIngestChannelInternal.java |  4 +-
 .../net/snowflake/ingest/utils/Logging.java   |  6 ++
 .../streaming/internal/RowBufferTest.java     | 81 ++++++++++++++++
 .../streaming/internal/StreamingIngestIT.java | 72 ++++++++++++--
 8 files changed, 244 insertions(+), 47 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 973b6e81b..0a8e3087d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -473,9 +473,7 @@ abstract float addTempRow(
   void reset() {
     this.rowCount = 0;
     this.bufferSize = 0F;
-    this.statsMap.replaceAll(
-        (key, value) ->
-            new RowBufferStats(value.getColumnDisplayName(), value.getCollationDefinitionString()));
+    this.statsMap.replaceAll((key, value) -> value.forkEmpty());
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 1adc14eda..c253014c0 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -425,16 +425,25 @@ private float convertRowToArrow(
       Set<String> inputColumnNames) {
     // Insert values to the corresponding arrow buffers
     float rowBufferSize = 0F;
+    // Create new empty stats just for the current row.
+    Map<String, RowBufferStats> forkedStatsMap = new HashMap<>();
+
+    // We need to iterate twice over the row and over unquoted names, we store the value to avoid
+    // re-computation
+    Map<String, String> userInputToUnquotedColumnNameMap = new HashMap<>();
+
     for (Map.Entry<String, Object> entry : row.entrySet()) {
       rowBufferSize += 0.125; // 1/8 for null value bitmap
       String columnName = LiteralQuoteUtils.unquoteColumnName(entry.getKey());
+      userInputToUnquotedColumnNameMap.put(entry.getKey(), columnName);
       Object value = entry.getValue();
       Field field = this.fields.get(columnName);
       Utils.assertNotNull("Arrow column field", field);
       FieldVector vector = sourceVectors.getVector(field);
       Utils.assertNotNull("Arrow column vector", vector);
-      RowBufferStats stats = statsMap.get(columnName);
-      Utils.assertNotNull("Arrow column stats", stats);
+      RowBufferStats forkedStats = statsMap.get(columnName).forkEmpty();
+      forkedStatsMap.put(columnName, forkedStats);
+      Utils.assertNotNull("Arrow column stats", forkedStats);
       ColumnLogicalType logicalType =
           ColumnLogicalType.valueOf(field.getMetadata().get(COLUMN_LOGICAL_TYPE));
       ColumnPhysicalType physicalType =
@@ -447,7 +456,8 @@ private float convertRowToArrow(
             int columnPrecision = Integer.parseInt(field.getMetadata().get(COLUMN_PRECISION));
             int columnScale = getColumnScale(field.getMetadata());
             BigDecimal inputAsBigDecimal =
-                DataValidationUtil.validateAndParseBigDecimal(stats.getColumnDisplayName(), value);
+                DataValidationUtil.validateAndParseBigDecimal(
+                    forkedStats.getColumnDisplayName(), value);
             // vector.setSafe requires the BigDecimal input scale explicitly match its scale
             inputAsBigDecimal = inputAsBigDecimal.setScale(columnScale, RoundingMode.HALF_UP);
 
@@ -455,29 +465,29 @@ private float convertRowToArrow(
 
             if (columnScale != 0 || physicalType == ColumnPhysicalType.SB16) {
               ((DecimalVector) vector).setSafe(curRowIndex, inputAsBigDecimal);
-              stats.addIntValue(inputAsBigDecimal.unscaledValue());
+              forkedStats.addIntValue(inputAsBigDecimal.unscaledValue());
               rowBufferSize += 16;
             } else {
               switch (physicalType) {
                 case SB1:
                   ((TinyIntVector) vector).setSafe(curRowIndex, inputAsBigDecimal.byteValueExact());
-                  stats.addIntValue(inputAsBigDecimal.toBigInteger());
+                  forkedStats.addIntValue(inputAsBigDecimal.toBigInteger());
                   rowBufferSize += 1;
                   break;
                 case SB2:
                   ((SmallIntVector) vector)
                       .setSafe(curRowIndex, inputAsBigDecimal.shortValueExact());
-                  stats.addIntValue(inputAsBigDecimal.toBigInteger());
+                  forkedStats.addIntValue(inputAsBigDecimal.toBigInteger());
                   rowBufferSize += 2;
                   break;
                 case SB4:
                   ((IntVector) vector).setSafe(curRowIndex, inputAsBigDecimal.intValueExact());
-                  stats.addIntValue(inputAsBigDecimal.toBigInteger());
+                  forkedStats.addIntValue(inputAsBigDecimal.toBigInteger());
                   rowBufferSize += 4;
                   break;
                 case SB8:
                   ((BigIntVector) vector).setSafe(curRowIndex, inputAsBigDecimal.longValueExact());
-                  stats.addIntValue(inputAsBigDecimal.toBigInteger());
+                  forkedStats.addIntValue(inputAsBigDecimal.toBigInteger());
                   rowBufferSize += 8;
                   break;
                 default:
@@ -492,19 +502,20 @@ private float convertRowToArrow(
               String maxLengthString = field.getMetadata().get(COLUMN_CHAR_LENGTH);
               String str =
                   DataValidationUtil.validateAndParseString(
-                      stats.getColumnDisplayName(),
+                      forkedStats.getColumnDisplayName(),
                       value,
                       Optional.ofNullable(maxLengthString).map(Integer::parseInt));
               Text text = new Text(str);
               ((VarCharVector) vector).setSafe(curRowIndex, text);
-              stats.addStrValue(str);
+              forkedStats.addStrValue(str);
               rowBufferSize += text.getBytes().length;
               break;
             }
           case OBJECT:
             {
               String str =
-                  DataValidationUtil.validateAndParseObject(stats.getColumnDisplayName(), value);
+                  DataValidationUtil.validateAndParseObject(
+                      forkedStats.getColumnDisplayName(), value);
               Text text = new Text(str);
               ((VarCharVector) vector).setSafe(curRowIndex, text);
               rowBufferSize += text.getBytes().length;
@@ -513,7 +524,8 @@ private float convertRowToArrow(
           case ARRAY:
             {
               String str =
-                  DataValidationUtil.validateAndParseArray(stats.getColumnDisplayName(), value);
+                  DataValidationUtil.validateAndParseArray(
+                      forkedStats.getColumnDisplayName(), value);
               Text text = new Text(str);
               ((VarCharVector) vector).setSafe(curRowIndex, text);
               rowBufferSize += text.getBytes().length;
@@ -522,7 +534,8 @@ private float convertRowToArrow(
           case VARIANT:
             {
               String str =
-                  DataValidationUtil.validateAndParseVariant(stats.getColumnDisplayName(), value);
+                  DataValidationUtil.validateAndParseVariant(
+                      forkedStats.getColumnDisplayName(), value);
               if (str != null) {
                 Text text = new Text(str);
                 ((VarCharVector) vector).setSafe(curRowIndex, text);
@@ -542,14 +555,14 @@ private float convertRowToArrow(
                   BigIntVector bigIntVector = (BigIntVector) vector;
                   TimestampWrapper timestampWrapper =
                       DataValidationUtil.validateAndParseTimestamp(
-                          stats.getColumnDisplayName(),
+                          forkedStats.getColumnDisplayName(),
                           value,
                           getColumnScale(field.getMetadata()),
                           defaultTimezone,
                           trimTimezone);
                   BigInteger timestampBinary = timestampWrapper.toBinary(false);
                   bigIntVector.setSafe(curRowIndex, timestampBinary.longValue());
-                  stats.addIntValue(timestampBinary);
+                  forkedStats.addIntValue(timestampBinary);
                   rowBufferSize += 8;
                   break;
                 }
@@ -565,7 +578,7 @@ private float convertRowToArrow(
 
                   TimestampWrapper timestampWrapper =
                       DataValidationUtil.validateAndParseTimestamp(
-                          stats.getColumnDisplayName(),
+                          forkedStats.getColumnDisplayName(),
                           value,
                           getColumnScale(field.getMetadata()),
                           defaultTimezone,
@@ -573,7 +586,7 @@ private float convertRowToArrow(
                   epochVector.setSafe(curRowIndex, timestampWrapper.getEpoch());
                   fractionVector.setSafe(curRowIndex, timestampWrapper.getFraction());
                   rowBufferSize += 12;
-                  stats.addIntValue(timestampWrapper.toBinary(false));
+                  forkedStats.addIntValue(timestampWrapper.toBinary(false));
                   break;
                 }
               default:
@@ -594,7 +607,7 @@ private float convertRowToArrow(
 
                   TimestampWrapper timestampWrapper =
                       DataValidationUtil.validateAndParseTimestamp(
-                          stats.getColumnDisplayName(),
+                          forkedStats.getColumnDisplayName(),
                           value,
                           getColumnScale(field.getMetadata()),
                           defaultTimezone,
@@ -603,7 +616,7 @@ private float convertRowToArrow(
                       curRowIndex, timestampWrapper.toBinary(false).longValueExact());
                   timezoneVector.setSafe(curRowIndex, timestampWrapper.getTimeZoneIndex());
                   rowBufferSize += 12;
-                  stats.addIntValue(timestampWrapper.toBinary(true));
+                  forkedStats.addIntValue(timestampWrapper.toBinary(true));
                   break;
                 }
               case SB16:
@@ -620,7 +633,7 @@ private float convertRowToArrow(
 
                   TimestampWrapper timestampWrapper =
                       DataValidationUtil.validateAndParseTimestamp(
-                          stats.getColumnDisplayName(),
+                          forkedStats.getColumnDisplayName(),
                           value,
                           getColumnScale(field.getMetadata()),
                           defaultTimezone,
@@ -630,7 +643,7 @@ private float convertRowToArrow(
                   timezoneVector.setSafe(curRowIndex, timestampWrapper.getTimeZoneIndex());
                   rowBufferSize += 16;
                   BigInteger timeInBinary = timestampWrapper.toBinary(true);
-                  stats.addIntValue(timeInBinary);
+                  forkedStats.addIntValue(timeInBinary);
                   break;
                 }
               default:
@@ -642,9 +655,10 @@ private float convertRowToArrow(
               DateDayVector dateDayVector = (DateDayVector) vector;
               // Expect days past the epoch
               int intValue =
-                  DataValidationUtil.validateAndParseDate(stats.getColumnDisplayName(), value);
+                  DataValidationUtil.validateAndParseDate(
+                      forkedStats.getColumnDisplayName(), value);
               dateDayVector.setSafe(curRowIndex, intValue);
-              stats.addIntValue(BigInteger.valueOf(intValue));
+              forkedStats.addIntValue(BigInteger.valueOf(intValue));
               rowBufferSize += 4;
               break;
             }
@@ -654,9 +668,11 @@ private float convertRowToArrow(
                 {
                   BigInteger timeInScale =
                       DataValidationUtil.validateAndParseTime(
-                          stats.getColumnDisplayName(), value, getColumnScale(field.getMetadata()));
+                          forkedStats.getColumnDisplayName(),
+                          value,
+                          getColumnScale(field.getMetadata()));
                   ((IntVector) vector).setSafe(curRowIndex, timeInScale.intValue());
-                  stats.addIntValue(timeInScale);
+                  forkedStats.addIntValue(timeInScale);
                   rowBufferSize += 4;
                   break;
                 }
@@ -664,9 +680,11 @@ private float convertRowToArrow(
                 {
                   BigInteger timeInScale =
                       DataValidationUtil.validateAndParseTime(
-                          stats.getColumnDisplayName(), value, getColumnScale(field.getMetadata()));
+                          forkedStats.getColumnDisplayName(),
+                          value,
+                          getColumnScale(field.getMetadata()));
                   ((BigIntVector) vector).setSafe(curRowIndex, timeInScale.longValue());
-                  stats.addIntValue(timeInScale);
+                  forkedStats.addIntValue(timeInScale);
                   rowBufferSize += 8;
                   break;
                 }
@@ -677,28 +695,29 @@ private float convertRowToArrow(
           case BOOLEAN:
             {
               int intValue =
-                  DataValidationUtil.validateAndParseBoolean(stats.getColumnDisplayName(), value);
+                  DataValidationUtil.validateAndParseBoolean(
+                      forkedStats.getColumnDisplayName(), value);
               ((BitVector) vector).setSafe(curRowIndex, intValue);
               rowBufferSize += 0.125;
-              stats.addIntValue(BigInteger.valueOf(intValue));
+              forkedStats.addIntValue(BigInteger.valueOf(intValue));
               break;
             }
           case BINARY:
             String maxLengthString = field.getMetadata().get(COLUMN_BYTE_LENGTH);
             byte[] bytes =
                 DataValidationUtil.validateAndParseBinary(
-                    stats.getColumnDisplayName(),
+                    forkedStats.getColumnDisplayName(),
                     value,
                     Optional.ofNullable(maxLengthString).map(Integer::parseInt));
             ((VarBinaryVector) vector).setSafe(curRowIndex, bytes);
-            stats.addBinaryValue(bytes);
+            forkedStats.addBinaryValue(bytes);
             rowBufferSize += bytes.length;
             break;
           case REAL:
             double doubleValue =
-                DataValidationUtil.validateAndParseReal(stats.getColumnDisplayName(), value);
+                DataValidationUtil.validateAndParseReal(forkedStats.getColumnDisplayName(), value);
             ((Float8Vector) vector).setSafe(curRowIndex, doubleValue);
-            stats.addRealValue(doubleValue);
+            forkedStats.addRealValue(doubleValue);
             rowBufferSize += 8;
             break;
           default:
@@ -711,11 +730,19 @@ private float convertRowToArrow(
           throw new SFException(
               ErrorCode.INVALID_ROW, columnName, "Passed null to non nullable field");
         } else {
-          insertNull(vector, stats, curRowIndex);
+          insertNull(vector, forkedStats, curRowIndex);
         }
       }
     }
 
+    // All input values passed validation, iterate over the columns again and combine their existing
+    // statistics with the forked statistics for the current row.
+    for (String userInputColumnName : row.keySet()) {
+      String columnName = userInputToUnquotedColumnNameMap.get(userInputColumnName);
+      RowBufferStats stats = statsMap.get(columnName);
+      RowBufferStats forkedStats = forkedStatsMap.get(columnName);
+      statsMap.put(columnName, RowBufferStats.getCombinedStats(stats, forkedStats));
+    }
     // Insert nulls to the columns that doesn't show up in the input
     for (String columnName : Sets.difference(this.fields.keySet(), inputColumnNames)) {
       rowBufferSize += 0.125; // 1/8 for null value bitmap
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index e3ea8d89d..bc53a61a7 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -179,24 +179,44 @@ private float addRow(
       Set<String> inputColumnNames) {
     Object[] indexedRow = new Object[fieldIndex.size()];
     float size = 0F;
+
+    // Create new empty stats just for the current row.
+    Map<String, RowBufferStats> forkedStatsMap = new HashMap<>();
+
+    // We need to iterate twice over the row and over unquoted names, we store the value to avoid
+    // re-computation
+    Map<String, String> userInputToUnquotedColumnNameMap = new HashMap<>();
+
     for (Map.Entry<String, Object> entry : row.entrySet()) {
       String key = entry.getKey();
       Object value = entry.getValue();
       String columnName = LiteralQuoteUtils.unquoteColumnName(key);
+      userInputToUnquotedColumnNameMap.put(key, columnName);
       int colIndex = fieldIndex.get(columnName).getSecond();
-      RowBufferStats stats = statsMap.get(columnName);
+      RowBufferStats forkedStats = statsMap.get(columnName).forkEmpty();
+      forkedStatsMap.put(columnName, forkedStats);
       ColumnMetadata column = fieldIndex.get(columnName).getFirst();
       ColumnDescriptor columnDescriptor = schema.getColumns().get(colIndex);
       PrimitiveType.PrimitiveTypeName typeName =
           columnDescriptor.getPrimitiveType().getPrimitiveTypeName();
       ParquetValueParser.ParquetBufferValue valueWithSize =
           ParquetValueParser.parseColumnValueToParquet(
-              value, column, typeName, stats, defaultTimezone);
+              value, column, typeName, forkedStats, defaultTimezone);
       indexedRow[colIndex] = valueWithSize.getValue();
       size += valueWithSize.getSize();
     }
     out.accept(Arrays.asList(indexedRow));
 
+    // All input values passed validation, iterate over the columns again and combine their existing
+    // statistics with the forked statistics for the current row.
+    for (String userInputColumnName : row.keySet()) {
+      String columnName = userInputToUnquotedColumnNameMap.get(userInputColumnName);
+      RowBufferStats stats = statsMap.get(columnName);
+      RowBufferStats forkedStats = forkedStatsMap.get(columnName);
+      statsMap.put(columnName, RowBufferStats.getCombinedStats(stats, forkedStats));
+    }
+
+    // Increment null count for column missing in the input map
     for (String columnName : Sets.difference(this.fieldIndex.keySet(), inputColumnNames)) {
       statsMap.get(columnName).incCurrentNullCount();
     }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
index 66848742b..88cc568d7 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
@@ -50,6 +50,11 @@ void reset() {
     this.currentMaxLength = 0;
   }
 
+  /** Create new statistics for the same column, with all calculated values set to empty */
+  RowBufferStats forkEmpty() {
+    return new RowBufferStats(this.getColumnDisplayName(), this.getCollationDefinitionString());
+  }
+
   // TODO performance test this vs in place update
   static RowBufferStats getCombinedStats(RowBufferStats left, RowBufferStats right) {
     if (!Objects.equals(left.getCollationDefinitionString(), right.collationDefinitionString)) {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 12cf0a15a..9a935b0a7 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -14,7 +14,7 @@
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.util.Collections;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -360,7 +360,7 @@ public InsertValidationResponse insertRows(
     // concurrently modified (e.g. byte[]). Before validation and EP calculation, we must make sure
     // that defensive copies of all mutable objects are created.
     final List<Map<String, Object>> rowsCopy = new LinkedList<>();
-    rows.forEach(r -> rowsCopy.add(new HashMap<>(r)));
+    rows.forEach(r -> rowsCopy.add(new LinkedHashMap<>(r)));
 
     InsertValidationResponse response = this.rowBuffer.insertRows(rowsCopy, offsetToken);
 
diff --git a/src/main/java/net/snowflake/ingest/utils/Logging.java b/src/main/java/net/snowflake/ingest/utils/Logging.java
index 2d5f90be0..1bd6c6b1a 100644
--- a/src/main/java/net/snowflake/ingest/utils/Logging.java
+++ b/src/main/java/net/snowflake/ingest/utils/Logging.java
@@ -71,6 +71,12 @@ public void logDebug(String format, Object... vars) {
     }
   }
 
+  public void logWarn(String format, Throwable e) {
+    if (log.isWarnEnabled()) {
+      log.warn(format, e);
+    }
+  }
+
   public void logWarn(String format, Object... vars) {
     if (log.isWarnEnabled()) {
       log.warn(format, vars);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 41aa793d3..16b8b67d5 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -10,6 +10,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
@@ -1063,6 +1064,86 @@ public void testExtraColumnsCheck() {
     Assert.assertEquals(Arrays.asList("COLBOOLEAN3", "COLBOOLEAN2"), error.getExtraColNames());
   }
 
+  @Test
+  public void testFailureHalfwayThroughColumnProcessingAbort() {
+    doTestFailureHalfwayThroughColumnProcessing(OpenChannelRequest.OnErrorOption.ABORT);
+  }
+
+  @Test
+  public void testFailureHalfwayThroughColumnProcessingContinue() {
+    doTestFailureHalfwayThroughColumnProcessing(OpenChannelRequest.OnErrorOption.CONTINUE);
+  }
+
+  private void doTestFailureHalfwayThroughColumnProcessing(
+      OpenChannelRequest.OnErrorOption onErrorOption) {
+    AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
+
+    ColumnMetadata colVarchar1 = new ColumnMetadata();
+    colVarchar1.setName("COLVARCHAR1");
+    colVarchar1.setPhysicalType("LOB");
+    colVarchar1.setNullable(true);
+    colVarchar1.setLogicalType("TEXT");
+    colVarchar1.setLength(1000);
+
+    ColumnMetadata colVarchar2 = new ColumnMetadata();
+    colVarchar2.setName("COLVARCHAR2");
+    colVarchar2.setPhysicalType("LOB");
+    colVarchar2.setNullable(true);
+    colVarchar2.setLogicalType("TEXT");
+    colVarchar2.setLength(1000);
+
+    ColumnMetadata colBoolean = new ColumnMetadata();
+    colBoolean.setName("COLBOOLEAN1");
+    colBoolean.setPhysicalType("SB1");
+    colBoolean.setNullable(true);
+    colBoolean.setLogicalType("BOOLEAN");
+    colBoolean.setScale(0);
+
+    innerBuffer.setupSchema(Arrays.asList(colVarchar1, colVarchar2, colBoolean));
+
+    LinkedHashMap<String, Object> row1 = new LinkedHashMap<>();
+    row1.put("COLVARCHAR1", null);
+    row1.put("COLVARCHAR2", "X");
+    row1.put("COLBOOLEAN1", "falze"); // will fail validation
+
+    LinkedHashMap<String, Object> row2 = new LinkedHashMap<>();
+    row2.put("COLVARCHAR1", "A");
+    row2.put("COLVARCHAR2", null);
+    row2.put("COLBOOLEAN1", "falze"); // will fail validation
+
+    LinkedHashMap<String, Object> row3 = new LinkedHashMap<>();
+    row3.put("COLVARCHAR1", "c");
+    row3.put("COLVARCHAR2", "d");
+    row3.put("COLBOOLEAN1", "true");
+
+    for (Map<String, Object> row : Arrays.asList(row1, row2, row3)) {
+      try {
+        innerBuffer.insertRows(Collections.singletonList(row), "");
+      } catch (Exception ignored) {
+        // we ignore exceptions, for ABORT option there will be some, but we don't care in this test
+      }
+    }
+
+    ChannelData<?> channelData = innerBuffer.flush("my_snowpipe_streaming.bdec");
+    RowBufferStats statsCol1 = channelData.getColumnEps().get("COLVARCHAR1");
+    RowBufferStats statsCol2 = channelData.getColumnEps().get("COLVARCHAR2");
+    RowBufferStats statsCol3 = channelData.getColumnEps().get("COLBOOLEAN1");
+    Assert.assertEquals(1, channelData.getRowCount());
+    Assert.assertEquals(0, statsCol1.getCurrentNullCount());
+    Assert.assertEquals(0, statsCol2.getCurrentNullCount());
+    Assert.assertEquals(0, statsCol3.getCurrentNullCount());
+    Assert.assertArrayEquals(
+        "c".getBytes(StandardCharsets.UTF_8), statsCol1.getCurrentMinStrValue());
+    Assert.assertArrayEquals(
+        "c".getBytes(StandardCharsets.UTF_8), statsCol1.getCurrentMaxStrValue());
+    Assert.assertArrayEquals(
+        "d".getBytes(StandardCharsets.UTF_8), statsCol2.getCurrentMinStrValue());
+    Assert.assertArrayEquals(
+        "d".getBytes(StandardCharsets.UTF_8), statsCol2.getCurrentMaxStrValue());
+    Assert.assertEquals(BigInteger.ONE, statsCol3.getCurrentMinIntValue());
+    Assert.assertEquals(BigInteger.ONE, statsCol3.getCurrentMaxIntValue());
+  }
+
   @Test
   public void testE2EBoolean() {
     testE2EBooleanHelper(OpenChannelRequest.OnErrorOption.ABORT);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 052fc021f..83a1a1e9e 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -17,6 +17,7 @@
 import java.util.Calendar;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
@@ -242,7 +243,13 @@ public void testInterleavedIngest() {
 
     SnowflakeStreamingIngestChannel[] channels =
         new SnowflakeStreamingIngestChannel[INTERLEAVED_CHANNEL_NUMBER];
-    iter.accept(i -> channels[i - 1] = openChannel(INTERLEAVED_TABLE_PREFIX + i, "CHANNEL"));
+    iter.accept(
+        i ->
+            channels[i - 1] =
+                openChannel(
+                    INTERLEAVED_TABLE_PREFIX + i,
+                    "CHANNEL",
+                    OpenChannelRequest.OnErrorOption.ABORT));
 
     iter.accept(
         i ->
@@ -266,7 +273,13 @@ public void testMultiChannelChunk() {
 
     SnowflakeStreamingIngestChannel[] channels =
         new SnowflakeStreamingIngestChannel[INTERLEAVED_CHANNEL_NUMBER];
-    iter.accept(i -> channels[i - 1] = openChannel(INTERLEAVED_CHANNEL_TABLE, "CHANNEL_" + i));
+    iter.accept(
+        i ->
+            channels[i - 1] =
+                openChannel(
+                    INTERLEAVED_CHANNEL_TABLE,
+                    "CHANNEL_" + i,
+                    OpenChannelRequest.OnErrorOption.ABORT));
 
     iter.accept(
         i ->
@@ -915,6 +928,50 @@ public void testNullValuesOnMultiDataTypes() throws Exception {
     }
   }
 
+  @Test
+  public void testFailureHalfwayThroughRowProcessing() throws Exception {
+    String tableName = "failure_halfway_through_table";
+    jdbcConnection
+        .createStatement()
+        .execute(
+            String.format("create or replace table %s(c1 varchar(1), c2 varchar(1));", tableName));
+    SnowflakeStreamingIngestChannel channel =
+        openChannel(tableName, "channel1", OpenChannelRequest.OnErrorOption.CONTINUE);
+
+    // Row will be ingested
+    Map<String, Object> row = new LinkedHashMap<>();
+    row.put("c1", null);
+    row.put("c2", "b");
+    channel.insertRow(row, "offset0");
+
+    // Row will not be ingested, its contribution to nullCount and other statistics should be
+    // discarded
+    row = new LinkedHashMap<>();
+    row.put("c1", null);
+    row.put("c2", "qa");
+    channel.insertRow(row, "offset1");
+
+    // Row will be ingested
+    row = new LinkedHashMap<>();
+    row.put("c1", null);
+    row.put("c2", null);
+    channel.insertRow(row, "offset2");
+
+    TestUtils.waitForOffset(channel, "offset2");
+
+    jdbcConnection.createStatement().execute(String.format("alter table %s migrate;", tableName));
+
+    ResultSet rs =
+        jdbcConnection.createStatement().executeQuery(String.format("select * from %s", tableName));
+    rs.next();
+    Assert.assertEquals(null, rs.getString(1));
+    Assert.assertEquals("b", rs.getString(2));
+
+    rs.next();
+    Assert.assertEquals(null, rs.getString(1));
+    Assert.assertEquals(null, rs.getString(2));
+  }
+
   @Test
   public void testColumnNameQuotes() throws Exception {
     final String tableName = "T_COLUMN_WITH_QUOTES";
@@ -1067,7 +1124,8 @@ private void ingestByColIndexAndVerifyTable(
     final int startIndex = (channelNum - 1) * numberOfRows;
 
     // ingest rows
-    SnowflakeStreamingIngestChannel channel = openChannel(tableName, channelName);
+    SnowflakeStreamingIngestChannel channel =
+        openChannel(tableName, channelName, OpenChannelRequest.OnErrorOption.ABORT);
     for (int i = 0; i < numberOfRows; i++) {
       Map<String, Object> row = new HashMap<>();
       for (int col : columnIndexes) {
@@ -1141,13 +1199,14 @@ private void createTableForInterleavedTest(String tableName) {
     }
   }
 
-  private SnowflakeStreamingIngestChannel openChannel(String tableName, String channelName) {
+  private SnowflakeStreamingIngestChannel openChannel(
+      String tableName, String channelName, OpenChannelRequest.OnErrorOption onErrorOption) {
     OpenChannelRequest request =
         OpenChannelRequest.builder(channelName)
             .setDBName(testDb)
             .setSchemaName(TEST_SCHEMA)
             .setTableName(tableName)
-            .setOnErrorOption(OpenChannelRequest.OnErrorOption.ABORT)
+            .setOnErrorOption(onErrorOption)
             .build();
 
     // Open a streaming ingest channel from the given client
@@ -1255,7 +1314,8 @@ public void testDataTypes() throws SQLException, ParseException {
       throw new RuntimeException("Cannot create table " + tableName, e);
     }
 
-    SnowflakeStreamingIngestChannel channel = openChannel(tableName, "CHANNEL");
+    SnowflakeStreamingIngestChannel channel =
+        openChannel(tableName, "CHANNEL", OpenChannelRequest.OnErrorOption.ABORT);
 
     Map<String, Object> negRow = getNegRow();
     verifyInsertValidationResponse(channel.insertRow(negRow, Integer.toString(0)));

From 9f44968b08af7abf39670666a5aab6c23fea54a8 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Fri, 24 Mar 2023 11:53:16 -0700
Subject: [PATCH 147/356] V1.1.1 release (#365)

V1.1.1 release
---
 pom.xml                                       | 1455 +++++++----------
 .../ingest/connection/RequestBuilder.java     |    2 +-
 2 files changed, 617 insertions(+), 840 deletions(-)

diff --git a/pom.xml b/pom.xml
index 2ae0cca98..de001de51 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,861 +1,638 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
 
-  <!-- Arifact name and version information -->
-  <groupId>net.snowflake</groupId>
-  <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>1.1.0</version>
-  <packaging>jar</packaging>
-  <name>Snowflake Ingest SDK</name>
-  <description>Snowflake Ingest SDK</description>
-  <url>https://www.snowflake.net/</url>
+    <!-- Arifact name and version information -->
+    <groupId>net.snowflake</groupId>
+    <artifactId>snowflake-ingest-sdk</artifactId>
+    <version>1.1.1</version>
+    <packaging>jar</packaging>
+    <name>Snowflake Ingest SDK</name>
+    <description>Snowflake Ingest SDK</description>
+    <url>https://www.snowflake.net/</url>
 
-  <licenses>
-    <license>
-      <name>The Apache Software License, Version 2.0</name>
-      <url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
-    </license>
-  </licenses>
 
-  <developers>
-    <developer>
-      <name>Snowflake Support Team</name>
-      <email>snowflake-java@snowflake.net</email>
-      <organization>Snowflake Computing</organization>
-      <organizationUrl>https://www.snowflake.net</organizationUrl>
-    </developer>
-  </developers>
+    <licenses>
+      <license>
+        <name>The Apache Software License, Version 2.0</name>
+        <url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
+      </license>
+    </licenses>
 
-  <scm>
-    <connection>scm:git:git://github.com/snowflakedb/snowflake-ingest-java</connection>
-    <url>https://github.com/snowflakedb/snowflake-ingest-java/tree/master</url>
-  </scm>
+    <developers>
+      <developer>
+        <name>Snowflake Support Team</name>
+        <email>snowflake-java@snowflake.net</email>
+        <organization>Snowflake Computing</organization>
+        <organizationUrl>https://www.snowflake.net</organizationUrl>
+      </developer>
+    </developers>
 
-  <!-- Set our Language Level to Java 8 -->
-  <properties>
-    <arrow.version>10.0.0</arrow.version>
-    <codehaus.version>1.9.13</codehaus.version>
-    <commonscodec.version>1.15</commonscodec.version>
-    <commonscollections.version>3.2.2</commonscollections.version>
-    <commonscompress.version>1.21</commonscompress.version>
-    <commonsio.version>2.8.0</commonsio.version>
-    <commonslang3.version>3.12.0</commonslang3.version>
-    <commonslogging.version>1.2</commonslogging.version>
-    <fasterxml.version>2.14.0</fasterxml.version>
-    <guava.version>31.1-jre</guava.version>
-    <hadoop.version>3.3.4</hadoop.version>
-    <jacoco.skip.instrument>true</jacoco.skip.instrument>
-    <jacoco.version>0.8.5</jacoco.version>
-    <maven.compiler.source>1.8</maven.compiler.source>
-    <maven.compiler.target>1.8</maven.compiler.target>
-    <netty.version>4.1.82.Final</netty.version>
-    <nimbusds.version>9.9.3</nimbusds.version>
-    <objenesis.version>3.1</objenesis.version>
-    <parquet.version>1.12.3</parquet.version>
-    <shadeBase>net.snowflake.ingest.internal</shadeBase>
-    <slf4j.version>1.7.36</slf4j.version>
-    <snappy.version>1.1.8.3</snappy.version>
-    <snowjdbc.version>3.13.15</snowjdbc.version>
-    <yetus.version>0.13.0</yetus.version>
-  </properties>
+    <scm>
+      <connection>scm:git:git://github.com/snowflakedb/snowflake-ingest-java</connection>
+      <url>https://github.com/snowflakedb/snowflake-ingest-java/tree/master</url>
+    </scm>
 
-  <dependencyManagement>
+    <!-- Set our Language Level to Java 8 -->
+    <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+        <shadeBase>net.snowflake.ingest.internal</shadeBase>
+        <jacoco.skip.instrument>true</jacoco.skip.instrument>
+        <jacoco.version>0.8.5</jacoco.version>
+        <arrow.version>10.0.0</arrow.version>
+        <hadoop.version>3.3.4</hadoop.version>
+    </properties>
+
+    <build>
+        <finalName>${project.artifactId}</finalName>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <filtering>true</filtering>
+            </resource>
+        </resources>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>2.19.1</version>
+                <configuration>
+                    <excludes>
+                        <exclude>**/TestSimpleIngestLocal.java</exclude>
+                    </excludes>
+                </configuration>
+            </plugin>
+            <plugin>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <addClasspath>true</addClasspath>
+                            <mainClass>net.snowflake.ingest.SimpleIngestManager</mainClass>
+                        </manifest>
+                    </archive>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>2.5.1</version>
+                <inherited>true</inherited>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>2.10.4</version>
+                <executions>
+                    <execution>
+                        <id>attach-javadocs</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                        <configuration>
+                            <additionalparam>-Xdoclint:none</additionalparam>
+                            <source>8</source>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <version>3.0.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>exec-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>check-shaded-content</id>
+                        <phase>verify</phase>
+                        <goals>
+                            <goal>exec</goal>
+                        </goals>
+                        <configuration>
+                            <executable>${basedir}/scripts/check_content.sh</executable>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+
+        <!-- disable default maven deploy plugin since we are using gpg:sign-and-deploy-file -->
+        <pluginManagement>
+            <plugins>
+                <plugin>
+                    <artifactId>maven-deploy-plugin</artifactId>
+                    <configuration>
+                        <skip>true</skip>
+                    </configuration>
+                </plugin>
+                <plugin>
+                    <groupId>org.jacoco</groupId>
+                    <artifactId>jacoco-maven-plugin</artifactId>
+                    <version>${jacoco.version}</version>
+                    <executions>
+                        <execution>
+                            <id>pre-unit-test</id>
+                            <goals>
+                                <goal>prepare-agent</goal>
+                            </goals>
+                            <configuration>
+                                <destFile>target/jacoco-ut.exec</destFile>
+                            </configuration>
+                        </execution>
+                        <execution>
+                            <id>post-unit-test</id>
+                            <phase>test</phase>
+                            <goals>
+                                <goal>report</goal>
+                            </goals>
+                            <configuration>
+                                <dataFile>target/jacoco-ut.exec</dataFile>
+                                <outputDirectory>target/jacoco-ut</outputDirectory>
+                            </configuration>
+                        </execution>
+                    </executions>
+                    <configuration>
+                        <skip>${jacoco.skip.instrument}</skip>
+                    </configuration>
+                </plugin>
+                <!-- Need to define the maven-failsafe-plugin here -->
+                <!-- http://maven.apache.org/surefire/maven-failsafe-plugin/plugin-info.html -->
+                <plugin>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-failsafe-plugin</artifactId>
+                    <version>3.0.0-M5</version>
+                </plugin>
+            </plugins>
+        </pluginManagement>
+    </build>
+
+    <!-- All of our needed dependencies -->
     <dependencies>
-      <dependency>
-        <groupId>com.fasterxml.jackson</groupId>
-        <artifactId>jackson-bom</artifactId>
-        <version>${fasterxml.version}</version>
-        <type>pom</type>
-        <scope>import</scope>
-      </dependency>
-      <dependency>
-        <groupId>com.google.guava</groupId>
-        <artifactId>guava</artifactId>
-        <version>${guava.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>com.nimbusds</groupId>
-        <artifactId>nimbus-jose-jwt</artifactId>
-        <version>${nimbusds.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>commons-codec</groupId>
-        <artifactId>commons-codec</artifactId>
-        <version>${commonscodec.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>commons-collections</groupId>
-        <artifactId>commons-collections</artifactId>
-        <version>${commonscollections.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>commons-io</groupId>
-        <artifactId>commons-io</artifactId>
-        <version>${commonsio.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>commons-logging</groupId>
-        <artifactId>commons-logging</artifactId>
-        <version>${commonslogging.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>io.netty</groupId>
-        <artifactId>netty-buffer</artifactId>
-        <version>${netty.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>io.netty</groupId>
-        <artifactId>netty-common</artifactId>
-        <version>${netty.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>net.snowflake</groupId>
-        <artifactId>snowflake-jdbc</artifactId>
-        <version>${snowjdbc.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.apache.commons</groupId>
-        <artifactId>commons-compress</artifactId>
-        <version>${commonscompress.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.apache.commons</groupId>
-        <artifactId>commons-lang3</artifactId>
-        <version>${commonslang3.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.apache.hadoop</groupId>
-        <artifactId>hadoop-common</artifactId>
-        <version>${hadoop.version}</version>
-        <exclusions>
-          <exclusion>
-            <groupId>ch.qos.reload4j</groupId>
-            <artifactId>reload4j</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>javax.activation</groupId>
-            <artifactId>activation</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-reload4j</artifactId>
-          </exclusion>
-        </exclusions>
-      </dependency>
-      <dependency>
-        <groupId>org.apache.parquet</groupId>
-        <artifactId>parquet-column</artifactId>
-        <version>${parquet.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.apache.parquet</groupId>
-        <artifactId>parquet-common</artifactId>
-        <version>${parquet.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.apache.parquet</groupId>
-        <artifactId>parquet-hadoop</artifactId>
-        <version>${parquet.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.apache.yetus</groupId>
-        <artifactId>audience-annotations</artifactId>
-        <version>${yetus.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-core-asl</artifactId>
-        <version>${codehaus.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-jaxrs</artifactId>
-        <version>${codehaus.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-mapper-asl</artifactId>
-        <version>${codehaus.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-xc</artifactId>
-        <version>${codehaus.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.codehaus.woodstox</groupId>
-        <artifactId>stax2-api</artifactId>
-        <version>4.2.1</version>
-      </dependency>
-      <dependency>
-        <groupId>org.objenesis</groupId>
-        <artifactId>objenesis</artifactId>
-        <version>${objenesis.version}</version>
-      </dependency>
-      <!-- SLF4J api that a client can shim in later -->
-      <dependency>
-        <groupId>org.slf4j</groupId>
-        <artifactId>slf4j-api</artifactId>
-        <version>${slf4j.version}</version>
-      </dependency>
-      <dependency>
-        <groupId>org.xerial.snappy</groupId>
-        <artifactId>snappy-java</artifactId>
-        <version>${snappy.version}</version>
-      </dependency>
 
-      <!-- JUnit so that we can make some basic unit tests -->
-      <dependency>
-        <groupId>junit</groupId>
-        <artifactId>junit</artifactId>
-        <version>4.13.2</version>
-        <scope>test</scope>
-      </dependency>
-      <dependency>
-        <groupId>net.bytebuddy</groupId>
-        <artifactId>byte-buddy</artifactId>
-        <version>1.10.19</version>
-        <scope>test</scope>
-      </dependency>
-      <dependency>
-        <groupId>net.bytebuddy</groupId>
-        <artifactId>byte-buddy-agent</artifactId>
-        <version>1.10.19</version>
-        <scope>test</scope>
-      </dependency>
-      <dependency>
-        <groupId>org.mockito</groupId>
-        <artifactId>mockito-core</artifactId>
-        <version>3.7.7</version>
-        <scope>test</scope>
-      </dependency>
-    </dependencies>
-  </dependencyManagement>
+        <!-- Snowflake JDBC used to connect to the service-->
+        <dependency>
+            <groupId>net.snowflake</groupId>
+            <artifactId>snowflake-jdbc</artifactId>
+            <version>3.13.15</version>
+        </dependency>
 
-  <!-- All of our needed dependencies -->
-  <dependencies>
-    <dependency>
-      <groupId>com.fasterxml.jackson.core</groupId>
-      <artifactId>jackson-annotations</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>com.fasterxml.jackson.core</groupId>
-      <artifactId>jackson-core</artifactId>
-    </dependency>
-    <!-- Jackson Databind api -->
-    <dependency>
-      <groupId>com.fasterxml.jackson.core</groupId>
-      <artifactId>jackson-databind</artifactId>
-    </dependency>
+        <!-- jwt token for key pair authentication with GS -->
+        <dependency>
+            <groupId>com.nimbusds</groupId>
+            <artifactId>nimbus-jose-jwt</artifactId>
+            <version>9.9.3</version>
+         </dependency>
 
-    <dependency>
-      <groupId>com.google.code.findbugs</groupId>
-      <artifactId>jsr305</artifactId>
-      <version>3.0.2</version>
-    </dependency>
-    <dependency>
-      <groupId>com.google.guava</groupId>
-      <artifactId>guava</artifactId>
-    </dependency>
+        <!-- Jackson Databind api -->
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.14.0</version>
+        </dependency>
 
-    <!-- jwt token for key pair authentication with GS -->
-    <dependency>
-      <groupId>com.nimbusds</groupId>
-      <artifactId>nimbus-jose-jwt</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>commons-codec</groupId>
-      <artifactId>commons-codec</artifactId>
-    </dependency>
+        <!-- SLF4J api that a client can shim in later -->
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.21</version>
+            <scope>provided</scope>
+        </dependency>
 
-    <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-core -->
-    <dependency>
-      <groupId>io.dropwizard.metrics</groupId>
-      <artifactId>metrics-core</artifactId>
-      <version>4.1.22</version>
-    </dependency>
+        <!-- JDK logger backend for logging tests -->
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-simple</artifactId>
+            <version>1.7.21</version>
+            <scope>test</scope>
+        </dependency>
 
-    <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-jmx -->
-    <dependency>
-      <groupId>io.dropwizard.metrics</groupId>
-      <artifactId>metrics-jmx</artifactId>
-      <version>4.1.22</version>
-    </dependency>
+        <!-- JUnit so that we can make some basic unit tests -->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.13.2</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-module-junit4</artifactId>
+            <version>2.0.2</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>3.7.7</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-api-mockito2</artifactId>
+            <version>2.0.2</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.powermock</groupId>
+            <artifactId>powermock-core</artifactId>
+            <version>2.0.2</version>
+            <scope>test</scope>
+        </dependency>
 
-    <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-jvm -->
-    <dependency>
-      <groupId>io.dropwizard.metrics</groupId>
-      <artifactId>metrics-jvm</artifactId>
-      <version>4.1.22</version>
-    </dependency>
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty-common</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>net.bytebuddy</groupId>
-      <artifactId>byte-buddy-agent</artifactId>
-    </dependency>
-    <!-- Snowflake JDBC used to connect to the service-->
-    <dependency>
-      <groupId>net.snowflake</groupId>
-      <artifactId>snowflake-jdbc</artifactId>
-    </dependency>
-    <!-- Apache Arrow -->
-    <dependency>
-      <groupId>org.apache.arrow</groupId>
-      <artifactId>arrow-memory-core</artifactId>
-      <version>${arrow.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.arrow</groupId>
-      <artifactId>arrow-vector</artifactId>
-      <version>${arrow.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-    </dependency>
-    <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop -->
-    <dependency>
-      <groupId>org.apache.parquet</groupId>
-      <artifactId>parquet-column</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.parquet</groupId>
-      <artifactId>parquet-common</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.parquet</groupId>
-      <artifactId>parquet-hadoop</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-api</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.arrow</groupId>
-      <artifactId>arrow-memory-netty</artifactId>
-      <version>${arrow.version}</version>
-      <scope>runtime</scope>
-    </dependency>
-    <!-- JUnit so that we can make some basic unit tests -->
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-lang3</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.hamcrest</groupId>
-      <artifactId>hamcrest-core</artifactId>
-      <version>1.3</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.mockito</groupId>
-      <artifactId>mockito-core</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.powermock</groupId>
-      <artifactId>powermock-api-mockito2</artifactId>
-      <version>2.0.2</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.powermock</groupId>
-      <artifactId>powermock-core</artifactId>
-      <version>2.0.2</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.powermock</groupId>
-      <artifactId>powermock-module-junit4</artifactId>
-      <version>2.0.2</version>
-      <scope>test</scope>
-    </dependency>
-  </dependencies>
+        <!-- Apache Arrow -->
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-vector</artifactId>
+            <version>${arrow.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-memory-netty</artifactId>
+            <version>${arrow.version}</version>
+            <scope>runtime</scope>
+        </dependency>
 
-  <build>
-    <finalName>${project.artifactId}</finalName>
-    <resources>
-      <resource>
-        <filtering>true</filtering>
-        <directory>src/main/resources</directory>
-      </resource>
-    </resources>
+        <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop -->
+        <dependency>
+            <groupId>org.apache.parquet</groupId>
+            <artifactId>parquet-hadoop</artifactId>
+            <version>1.12.3</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>javax.annotation</groupId>
+                    <artifactId>javax.annotation-api</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
 
-    <!-- disable default maven deploy plugin since we are using gpg:sign-and-deploy-file -->
-    <pluginManagement>
-      <plugins>
-        <plugin>
-          <artifactId>maven-deploy-plugin</artifactId>
-          <configuration>
-            <skip>true</skip>
-          </configuration>
-        </plugin>
-        <!-- Need to define the maven-failsafe-plugin here -->
-        <!-- http://maven.apache.org/surefire/maven-failsafe-plugin/plugin-info.html -->
-        <plugin>
-          <groupId>org.apache.maven.plugins</groupId>
-          <artifactId>maven-failsafe-plugin</artifactId>
-          <version>3.0.0-M5</version>
-        </plugin>
-        <plugin>
-          <groupId>org.jacoco</groupId>
-          <artifactId>jacoco-maven-plugin</artifactId>
-          <version>${jacoco.version}</version>
-          <configuration>
-            <skip>${jacoco.skip.instrument}</skip>
-          </configuration>
-          <executions>
-            <execution>
-              <id>pre-unit-test</id>
-              <goals>
-                <goal>prepare-agent</goal>
-              </goals>
-              <configuration>
-                <destFile>target/jacoco-ut.exec</destFile>
-              </configuration>
-            </execution>
-            <execution>
-              <id>post-unit-test</id>
-              <goals>
-                <goal>report</goal>
-              </goals>
-              <phase>test</phase>
-              <configuration>
-                <dataFile>target/jacoco-ut.exec</dataFile>
-                <outputDirectory>target/jacoco-ut</outputDirectory>
-              </configuration>
-            </execution>
-          </executions>
-        </plugin>
-      </plugins>
-    </pluginManagement>
-    <plugins>
-      <plugin>
-        <groupId>com.github.ekryd.sortpom</groupId>
-        <artifactId>sortpom-maven-plugin</artifactId>
-        <version>3.0.1</version>
-        <configuration>
-          <createBackupFile>false</createBackupFile>
-          <expandEmptyElements>false</expandEmptyElements>
-          <indentSchemaLocation>true</indentSchemaLocation>
-          <sortDependencies>scope,groupId,artifactId</sortDependencies>
-          <sortDependencyExclusions>groupId,artifactId</sortDependencyExclusions>
-          <sortExecutions>true</sortExecutions>
-          <sortModules>true</sortModules>
-          <sortPlugins>groupId,artifactId</sortPlugins>
-          <sortProperties>true</sortProperties>
-          <verifyFail>stop</verifyFail>
-          <verifyFailOn>strict</verifyFailOn>
-        </configuration>
-        <executions>
-          <execution>
-            <goals>
-              <goal>verify</goal>
-            </goals>
-            <phase>validate</phase>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <artifactId>maven-assembly-plugin</artifactId>
-        <configuration>
-          <archive>
-            <manifest>
-              <addClasspath>true</addClasspath>
-              <mainClass>net.snowflake.ingest.SimpleIngestManager</mainClass>
-            </manifest>
-          </archive>
-          <descriptorRefs>
-            <descriptorRef>jar-with-dependencies</descriptorRef>
-          </descriptorRefs>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-compiler-plugin</artifactId>
-        <version>2.5.1</version>
-        <inherited>true</inherited>
-        <configuration>
-          <source>1.8</source>
-          <target>1.8</target>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-dependency-plugin</artifactId>
-        <version>3.3.0</version>
-        <executions>
-          <execution>
-            <id>analyze</id>
-            <goals>
-              <goal>analyze-only</goal>
-            </goals>
-            <configuration>
-              <failOnWarning>true</failOnWarning>
-              <ignoreNonCompile>true</ignoreNonCompile>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-enforcer-plugin</artifactId>
-        <version>3.0.0-M3</version>
-        <dependencies>
-          <dependency>
-            <groupId>com.google.cloud.tools</groupId>
-            <artifactId>linkage-checker-enforcer-rules</artifactId>
-            <version>1.5.12</version>
-          </dependency>
-          <dependency>
-            <groupId>org.codehaus.mojo</groupId>
-            <artifactId>extra-enforcer-rules</artifactId>
-            <version>1.3</version>
+        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-mapreduce-client-core</artifactId>
+            <version>${hadoop.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-yarn-common</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-hdfs-client</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.avro</groupId>
+                    <artifactId>avro</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.inject.extensions</groupId>
+                    <artifactId>guice-servlet</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.eclipse.jetty.websocket</groupId>
+                    <artifactId>websocket-client</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>io.netty</groupId>
+                    <artifactId>netty</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-reload4j</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>${hadoop.version}</version>
             <exclusions>
-              <exclusion>
-                <groupId>org.eclipse.aether</groupId>
-                <artifactId>aether-util</artifactId>
-              </exclusion>
+                <exclusion>
+                    <groupId>com.github.spotbugs</groupId>
+                    <artifactId>spotbugs-annotations</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.nimbusds</groupId>
+                    <artifactId>nimbus-jose-jwt</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-json</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-server</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.sun.jersey</groupId>
+                    <artifactId>jersey-servlet</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>javax.servlet</groupId>
+                    <artifactId>servlet-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>javax.servlet</groupId>
+                    <artifactId>javax.servlet-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>javax.servlet.jsp</groupId>
+                    <artifactId>jsp-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.codehaus.jackson</groupId>
+                    <artifactId>jackson-mapper-asl</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.mortbay.jetty</groupId>
+                    <artifactId>jetty</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.eclipse.jetty</groupId>
+                    <artifactId>jetty-server</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.zookeeper</groupId>
+                    <artifactId>zookeeper</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.avro</groupId>
+                    <artifactId>avro</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-reload4j</artifactId>
+                </exclusion>
             </exclusions>
-          </dependency>
-        </dependencies>
-        <executions>
-          <execution>
-            <id>enforce-best-practices</id>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <banDuplicateClasses>
-                  <findAllDuplicates>true</findAllDuplicates>
-                  <ignoreWhenIdentical>true</ignoreWhenIdentical>
-                </banDuplicateClasses>
-                <banDuplicatePomDependencyVersions/>
-                <bannedDependencies>
-                  <excludes>
-                    <!-- <exclude>org.bouncycastle:bcprov-jdk15on</exclude> -->
-                  </excludes>
-                </bannedDependencies>
-                <dependencyConvergence/>
-                <requireUpperBoundDeps/>
-              </rules>
-            </configuration>
-          </execution>
-          <execution>
-            <id>enforce-maven</id>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <configuration>
-              <rules>
-                <requireMavenVersion>
-                  <version>3.5</version>
-                </requireMavenVersion>
-              </rules>
-            </configuration>
-          </execution>
-          <execution>
-            <id>enforce-linkage-checker</id>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <phase>verify</phase>
-            <configuration>
-              <rules>
-                <LinkageCheckerRule implementation="com.google.cloud.tools.dependencies.enforcer.LinkageCheckerRule">
-                  <reportOnlyReachable>true</reportOnlyReachable>
-                  <exclusionFile>${basedir}/linkage-checker-exclusion-rules.xml</exclusionFile>
-                  <level>WARN</level>
-                </LinkageCheckerRule>
-                <bannedDependencies>
-                  <excludes>
-                    <!-- JUnit 4 is not allowed on the classpath to prevent its accidental use -->
-                    <!-- <exclude>junit:junit</exclude> -->
-                  </excludes>
-                </bannedDependencies>
-              </rules>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-javadoc-plugin</artifactId>
-        <version>2.10.4</version>
-        <executions>
-          <execution>
-            <id>attach-javadocs</id>
-            <goals>
-              <goal>jar</goal>
-            </goals>
-            <configuration>
-              <additionalparam>-Xdoclint:none</additionalparam>
-              <source>8</source>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-source-plugin</artifactId>
-        <version>3.0.1</version>
-        <executions>
-          <execution>
-            <id>attach-sources</id>
-            <goals>
-              <goal>jar</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-surefire-plugin</artifactId>
-        <version>2.19.1</version>
-        <configuration>
-          <excludes>
-            <exclude>**/TestSimpleIngestLocal.java</exclude>
-          </excludes>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>exec-maven-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>check-shaded-content</id>
-            <goals>
-              <goal>exec</goal>
-            </goals>
-            <phase>verify</phase>
-            <configuration>
-              <executable>${basedir}/scripts/check_content.sh</executable>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-    </plugins>
-  </build>
+        </dependency>
 
-  <profiles>
-    <profile>
-      <id>shadeDep</id>
-      <activation>
-        <property>
-          <name>!not-shadeDep</name>
-        </property>
-      </activation>
-      <build>
-        <plugins>
-          <!-- Relocate all dependencies to internal to solve dependency conflict problem -->
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-shade-plugin</artifactId>
-            <version>3.1.1</version>
-            <configuration>
-              <relocations>
-                <relocation>
-                  <pattern>com.nimbusds</pattern>
-                  <shadedPattern>${shadeBase}.com.nimbusds</shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>net.jcip</pattern>
-                  <shadedPattern>${shadeBase}.net.jcip</shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>net.minidev</pattern>
-                  <shadedPattern>${shadeBase}.net.minidev</shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>org.objectweb</pattern>
-                  <shadedPattern>${shadeBase}.org.objectweb</shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>com.fasterxml</pattern>
-                  <shadedPattern>${shadeBase}.fasterxml</shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>org.apache</pattern>
-                  <shadedPattern>${shadeBase}.apache</shadedPattern>
-                </relocation>
-              </relocations>
-              <filters>
-                <filter>
-                  <artifact>*:*</artifact>
-                  <excludes>
-                    <exclude>META-INF/LICENSE*</exclude>
-                    <exclude>META-INF/NOTICE*</exclude>
-                    <exclude>META-INF/DEPENDENCIES</exclude>
-                    <exclude>META-INF/maven/**</exclude>
-                    <exclude>META-INF/services/com.fasterxml.*</exclude>
-                    <exclude>META-INF/*.xml</exclude>
-                    <exclude>META-INF/*.SF</exclude>
-                    <exclude>META-INF/*.DSA</exclude>
-                    <exclude>META-INF/*.RSA</exclude>
-                  </excludes>
-                </filter>
-                <filter>
-                  <artifact>commons-logging:commons-logging</artifact>
-                  <excludes>
-                    <exclude>org/apache/commons/logging/impl/AvalonLogger.class</exclude>
-                  </excludes>
-                </filter>
-                <filter>
-                  <artifact>org.slf4j:slf4j-simple</artifact>
-                  <excludes>
-                    <exclude>org/slf4j/**</exclude>
-                  </excludes>
-                </filter>
-              </filters>
-              <transformers>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"/>
-              </transformers>
-            </configuration>
-            <executions>
-              <execution>
-                <goals>
-                  <goal>shade</goal>
-                </goals>
-                <phase>package</phase>
-              </execution>
-            </executions>
-          </plugin>
-        </plugins>
-      </build>
-    </profile>
-    <profile>
-      <id>ossrh-deploy</id>
-      <activation>
-        <property>
-          <name>ossrh-deploy</name>
-        </property>
-      </activation>
-      <build>
-        <plugins>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-gpg-plugin</artifactId>
-            <version>1.6</version>
-            <executions>
-              <execution>
-                <goals>
-                  <goal>sign-and-deploy-file</goal>
-                </goals>
-                <phase>deploy</phase>
-                <configuration>
-                  <file>target/${project.artifactId}.jar</file>
-                  <repositoryId>ossrh</repositoryId>
-                  <url>https://oss.sonatype.org/service/local/staging/deploy/maven2</url>
-                  <pomFile>generated_public_pom.xml</pomFile>
-                  <javadoc>target/${project.artifactId}-javadoc.jar</javadoc>
-                  <sources>target/${project.artifactId}-sources.jar</sources>
-                  <keyname>${env.GPG_KEY_ID}</keyname>
-                  <passphrase>${env.GPG_KEY_PASSPHRASE}</passphrase>
-                </configuration>
-              </execution>
-            </executions>
-          </plugin>
-        </plugins>
-      </build>
-    </profile>
-    <profile>
-      <id>ghActionsIT</id>
-      <activation>
-        <property>
-          <name>ghActionsIT</name>
-        </property>
-      </activation>
-      <build>
-        <plugins>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-failsafe-plugin</artifactId>
-            <executions>
-              <execution>
-                <goals>
-                  <goal>integration-test</goal>
-                </goals>
-              </execution>
-              <execution>
-                <id>verify_github_actions_it</id>
-                <goals>
-                  <goal>verify</goal>
-                </goals>
-                <phase>verify</phase>
-              </execution>
-            </executions>
-          </plugin>
-          <plugin>
-            <groupId>org.jacoco</groupId>
-            <artifactId>jacoco-maven-plugin</artifactId>
-            <version>${jacoco.version}</version>
-            <configuration>
-              <skip>${jacoco.skip.instrument}</skip>
-            </configuration>
-            <executions>
-              <execution>
-                <id>pre-unit-test</id>
-                <goals>
-                  <goal>prepare-agent</goal>
-                </goals>
-                <configuration>
-                  <destFile>target/jacoco-ut.exec</destFile>
-                </configuration>
-              </execution>
-              <execution>
-                <id>post-unit-test</id>
-                <goals>
-                  <goal>report</goal>
-                </goals>
-                <phase>test</phase>
-                <configuration>
-                  <dataFile>target/jacoco-ut.exec</dataFile>
-                  <outputDirectory>target/jacoco-ut</outputDirectory>
-                </configuration>
-              </execution>
-              <execution>
-                <id>pre-integration-test</id>
-                <goals>
-                  <goal>prepare-agent</goal>
-                </goals>
-                <phase>pre-integration-test</phase>
-                <configuration>
-                  <destFile>target/jacoco-it.exec</destFile>
-                </configuration>
-              </execution>
-              <execution>
-                <id>post-integration-test</id>
-                <goals>
-                  <goal>report</goal>
-                </goals>
-                <phase>post-integration-test</phase>
-                <configuration>
-                  <dataFile>target/jacoco-it.exec</dataFile>
-                  <outputDirectory>target/jacoco-it</outputDirectory>
-                </configuration>
-              </execution>
-            </executions>
-          </plugin>
-        </plugins>
-      </build>
-    </profile>
-  </profiles>
+        <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-core -->
+        <dependency>
+            <groupId>io.dropwizard.metrics</groupId>
+            <artifactId>metrics-core</artifactId>
+            <version>4.1.22</version>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-jvm -->
+        <dependency>
+            <groupId>io.dropwizard.metrics</groupId>
+            <artifactId>metrics-jvm</artifactId>
+            <version>4.1.22</version>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-jmx -->
+        <dependency>
+            <groupId>io.dropwizard.metrics</groupId>
+            <artifactId>metrics-jmx</artifactId>
+            <version>4.2.3</version>
+        </dependency>
+    </dependencies>
+
+    <profiles>
+        <profile>
+            <id>shadeDep</id>
+            <activation>
+                <property>
+                    <name>!not-shadeDep</name>
+                </property>
+            </activation>
+            <build>
+                <plugins>
+                    <!-- Relocate all dependencies to internal to solve dependency conflict problem -->
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-shade-plugin</artifactId>
+                        <version>3.1.1</version>
+                        <configuration>
+                            <relocations>
+                                <relocation>
+                                    <pattern>com.nimbusds</pattern>
+                                    <shadedPattern>
+                                        ${shadeBase}.com.nimbusds
+                                    </shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>net.jcip</pattern>
+                                    <shadedPattern>${shadeBase}.net.jcip</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>net.minidev</pattern>
+                                    <shadedPattern>${shadeBase}.net.minidev</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>org.objectweb</pattern>
+                                    <shadedPattern>${shadeBase}.org.objectweb</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>com.fasterxml</pattern>
+                                    <shadedPattern>${shadeBase}.fasterxml</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>org.apache</pattern>
+                                    <shadedPattern>${shadeBase}.apache</shadedPattern>
+                                </relocation>
+                            </relocations>
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/LICENSE*</exclude>
+                                        <exclude>META-INF/NOTICE*</exclude>
+                                        <exclude>META-INF/DEPENDENCIES</exclude>
+                                        <exclude>META-INF/maven/**</exclude>
+                                        <exclude>META-INF/services/com.fasterxml.*</exclude>
+                                        <exclude>META-INF/*.xml</exclude>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                                <filter>
+                                    <artifact>commons-logging:commons-logging</artifact>
+                                    <excludes>
+                                        <exclude>org/apache/commons/logging/impl/AvalonLogger.class</exclude>
+                                    </excludes>
+                                </filter>
+                                <filter>
+                                    <artifact>org.slf4j:slf4j-simple</artifact>
+                                    <excludes>
+                                        <exclude>org/slf4j/**</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                        <executions>
+                            <execution>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>shade</goal>
+                                </goals>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+        <profile>
+            <id>ossrh-deploy</id>
+            <activation>
+                <property>
+                    <name>ossrh-deploy</name>
+                </property>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-gpg-plugin</artifactId>
+                        <version>1.6</version>
+                        <executions>
+                            <execution>
+                                <phase>deploy</phase>
+                                <goals>
+                                    <goal>sign-and-deploy-file</goal>
+                                </goals>
+                                <configuration>
+                                    <file>target/${project.artifactId}.jar</file>
+                                    <repositoryId>ossrh</repositoryId>
+                                    <url>https://oss.sonatype.org/service/local/staging/deploy/maven2</url>
+                                    <pomFile>generated_public_pom.xml</pomFile>
+                                    <javadoc>target/${project.artifactId}-javadoc.jar</javadoc>
+                                    <sources>target/${project.artifactId}-sources.jar</sources>
+                                    <keyname>${env.GPG_KEY_ID}</keyname>
+                                    <passphrase>${env.GPG_KEY_PASSPHRASE}</passphrase>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+        <profile>
+            <id>ghActionsIT</id>
+            <activation>
+                <property>
+                    <name>ghActionsIT</name>
+                </property>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-failsafe-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <goals>
+                                    <goal>integration-test</goal>
+                                </goals>
+                            </execution>
+                            <execution>
+                                <id>verify_github_actions_it</id>
+                                <phase>verify</phase>
+                                <goals>
+                                    <goal>verify</goal>
+                                </goals>
+                            </execution>
+                        </executions>
+                    </plugin>
+                    <plugin>
+                        <groupId>org.jacoco</groupId>
+                        <artifactId>jacoco-maven-plugin</artifactId>
+                        <version>${jacoco.version}</version>
+                        <executions>
+                            <execution>
+                                <id>pre-unit-test</id>
+                                <goals>
+                                    <goal>prepare-agent</goal>
+                                </goals>
+                                <configuration>
+                                    <destFile>target/jacoco-ut.exec</destFile>
+                                </configuration>
+                            </execution>
+                            <execution>
+                                <id>post-unit-test</id>
+                                <phase>test</phase>
+                                <goals>
+                                    <goal>report</goal>
+                                </goals>
+                                <configuration>
+                                    <dataFile>target/jacoco-ut.exec</dataFile>
+                                    <outputDirectory>target/jacoco-ut</outputDirectory>
+                                </configuration>
+                            </execution>
+                            <execution>
+                                <id>pre-integration-test</id>
+                                <phase>pre-integration-test</phase>
+                                <goals>
+                                    <goal>prepare-agent</goal>
+                                </goals>
+                                <configuration>
+                                    <destFile>target/jacoco-it.exec</destFile>
+                                </configuration>
+                            </execution>
+                            <execution>
+                                <id>post-integration-test</id>
+                                <phase>post-integration-test</phase>
+                                <goals>
+                                    <goal>report</goal>
+                                </goals>
+                                <configuration>
+                                    <dataFile>target/jacoco-it.exec</dataFile>
+                                    <outputDirectory>target/jacoco-it</outputDirectory>
+                                </configuration>
+                            </execution>
+                        </executions>
+                        <configuration>
+                            <skip>${jacoco.skip.instrument}</skip>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
 
 </project>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 38c8a0e0d..746f8122d 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -127,7 +127,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.1.0";
+  public static final String DEFAULT_VERSION = "1.1.1";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 893a996ec2b18768e13a25305e2e551a4f4d11d9 Mon Sep 17 00:00:00 2001
From: David Wang <david.wang@snowflake.com>
Date: Mon, 27 Mar 2023 23:23:25 +0000
Subject: [PATCH 148/356] SNOW-765525 fix build error

---
 pom.xml | 1451 ++++++++++++++++++++++++++++++++-----------------------
 1 file changed, 835 insertions(+), 616 deletions(-)

diff --git a/pom.xml b/pom.xml
index de001de51..884450447 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,638 +1,857 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
 
-    <!-- Arifact name and version information -->
-    <groupId>net.snowflake</groupId>
-    <artifactId>snowflake-ingest-sdk</artifactId>
-    <version>1.1.1</version>
-    <packaging>jar</packaging>
-    <name>Snowflake Ingest SDK</name>
-    <description>Snowflake Ingest SDK</description>
-    <url>https://www.snowflake.net/</url>
+  <!-- Arifact name and version information -->
+  <groupId>net.snowflake</groupId>
+  <artifactId>snowflake-ingest-sdk</artifactId>
+  <version>1.1.1</version>
+  <packaging>jar</packaging>
+  <name>Snowflake Ingest SDK</name>
+  <description>Snowflake Ingest SDK</description>
+  <url>https://www.snowflake.net/</url>
 
+  <licenses>
+    <license>
+      <name>The Apache Software License, Version 2.0</name>
+      <url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
+    </license>
+  </licenses>
 
-    <licenses>
-      <license>
-        <name>The Apache Software License, Version 2.0</name>
-        <url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
-      </license>
-    </licenses>
+  <developers>
+    <developer>
+      <name>Snowflake Support Team</name>
+      <email>snowflake-java@snowflake.net</email>
+      <organization>Snowflake Computing</organization>
+      <organizationUrl>https://www.snowflake.net</organizationUrl>
+    </developer>
+  </developers>
 
-    <developers>
-      <developer>
-        <name>Snowflake Support Team</name>
-        <email>snowflake-java@snowflake.net</email>
-        <organization>Snowflake Computing</organization>
-        <organizationUrl>https://www.snowflake.net</organizationUrl>
-      </developer>
-    </developers>
+  <scm>
+    <connection>scm:git:git://github.com/snowflakedb/snowflake-ingest-java</connection>
+    <url>https://github.com/snowflakedb/snowflake-ingest-java/tree/master</url>
+  </scm>
 
-    <scm>
-      <connection>scm:git:git://github.com/snowflakedb/snowflake-ingest-java</connection>
-      <url>https://github.com/snowflakedb/snowflake-ingest-java/tree/master</url>
-    </scm>
+  <!-- Set our Language Level to Java 8 -->
+  <properties>
+    <arrow.version>10.0.0</arrow.version>
+    <codehaus.version>1.9.13</codehaus.version>
+    <commonscodec.version>1.15</commonscodec.version>
+    <commonscollections.version>3.2.2</commonscollections.version>
+    <commonscompress.version>1.21</commonscompress.version>
+    <commonsio.version>2.8.0</commonsio.version>
+    <commonslang3.version>3.12.0</commonslang3.version>
+    <commonslogging.version>1.2</commonslogging.version>
+    <fasterxml.version>2.14.0</fasterxml.version>
+    <guava.version>31.1-jre</guava.version>
+    <hadoop.version>3.3.4</hadoop.version>
+    <jacoco.skip.instrument>true</jacoco.skip.instrument>
+    <jacoco.version>0.8.5</jacoco.version>
+    <maven.compiler.source>1.8</maven.compiler.source>
+    <maven.compiler.target>1.8</maven.compiler.target>
+    <netty.version>4.1.82.Final</netty.version>
+    <nimbusds.version>9.9.3</nimbusds.version>
+    <objenesis.version>3.1</objenesis.version>
+    <parquet.version>1.12.3</parquet.version>
+    <shadeBase>net.snowflake.ingest.internal</shadeBase>
+    <slf4j.version>1.7.36</slf4j.version>
+    <snappy.version>1.1.8.3</snappy.version>
+    <snowjdbc.version>3.13.15</snowjdbc.version>
+    <yetus.version>0.13.0</yetus.version>
+  </properties>
 
-    <!-- Set our Language Level to Java 8 -->
-    <properties>
-        <maven.compiler.source>1.8</maven.compiler.source>
-        <maven.compiler.target>1.8</maven.compiler.target>
-        <shadeBase>net.snowflake.ingest.internal</shadeBase>
-        <jacoco.skip.instrument>true</jacoco.skip.instrument>
-        <jacoco.version>0.8.5</jacoco.version>
-        <arrow.version>10.0.0</arrow.version>
-        <hadoop.version>3.3.4</hadoop.version>
-    </properties>
-
-    <build>
-        <finalName>${project.artifactId}</finalName>
-        <resources>
-            <resource>
-                <directory>src/main/resources</directory>
-                <filtering>true</filtering>
-            </resource>
-        </resources>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.19.1</version>
-                <configuration>
-                    <excludes>
-                        <exclude>**/TestSimpleIngestLocal.java</exclude>
-                    </excludes>
-                </configuration>
-            </plugin>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <configuration>
-                    <archive>
-                        <manifest>
-                            <addClasspath>true</addClasspath>
-                            <mainClass>net.snowflake.ingest.SimpleIngestManager</mainClass>
-                        </manifest>
-                    </archive>
-                    <descriptorRefs>
-                        <descriptorRef>jar-with-dependencies</descriptorRef>
-                    </descriptorRefs>
-                </configuration>
-            </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-compiler-plugin</artifactId>
-                <version>2.5.1</version>
-                <inherited>true</inherited>
-                <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
-                </configuration>
-            </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-javadoc-plugin</artifactId>
-                <version>2.10.4</version>
-                <executions>
-                    <execution>
-                        <id>attach-javadocs</id>
-                        <goals>
-                            <goal>jar</goal>
-                        </goals>
-                        <configuration>
-                            <additionalparam>-Xdoclint:none</additionalparam>
-                            <source>8</source>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-source-plugin</artifactId>
-                <version>3.0.1</version>
-                <executions>
-                    <execution>
-                        <id>attach-sources</id>
-                        <goals>
-                            <goal>jar</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>exec-maven-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>check-shaded-content</id>
-                        <phase>verify</phase>
-                        <goals>
-                            <goal>exec</goal>
-                        </goals>
-                        <configuration>
-                            <executable>${basedir}/scripts/check_content.sh</executable>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-
-        <!-- disable default maven deploy plugin since we are using gpg:sign-and-deploy-file -->
-        <pluginManagement>
-            <plugins>
-                <plugin>
-                    <artifactId>maven-deploy-plugin</artifactId>
-                    <configuration>
-                        <skip>true</skip>
-                    </configuration>
-                </plugin>
-                <plugin>
-                    <groupId>org.jacoco</groupId>
-                    <artifactId>jacoco-maven-plugin</artifactId>
-                    <version>${jacoco.version}</version>
-                    <executions>
-                        <execution>
-                            <id>pre-unit-test</id>
-                            <goals>
-                                <goal>prepare-agent</goal>
-                            </goals>
-                            <configuration>
-                                <destFile>target/jacoco-ut.exec</destFile>
-                            </configuration>
-                        </execution>
-                        <execution>
-                            <id>post-unit-test</id>
-                            <phase>test</phase>
-                            <goals>
-                                <goal>report</goal>
-                            </goals>
-                            <configuration>
-                                <dataFile>target/jacoco-ut.exec</dataFile>
-                                <outputDirectory>target/jacoco-ut</outputDirectory>
-                            </configuration>
-                        </execution>
-                    </executions>
-                    <configuration>
-                        <skip>${jacoco.skip.instrument}</skip>
-                    </configuration>
-                </plugin>
-                <!-- Need to define the maven-failsafe-plugin here -->
-                <!-- http://maven.apache.org/surefire/maven-failsafe-plugin/plugin-info.html -->
-                <plugin>
-                    <groupId>org.apache.maven.plugins</groupId>
-                    <artifactId>maven-failsafe-plugin</artifactId>
-                    <version>3.0.0-M5</version>
-                </plugin>
-            </plugins>
-        </pluginManagement>
-    </build>
-
-    <!-- All of our needed dependencies -->
+  <dependencyManagement>
     <dependencies>
+      <dependency>
+        <groupId>com.fasterxml.jackson</groupId>
+        <artifactId>jackson-bom</artifactId>
+        <version>${fasterxml.version}</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
+      <dependency>
+        <groupId>com.google.guava</groupId>
+        <artifactId>guava</artifactId>
+        <version>${guava.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>com.nimbusds</groupId>
+        <artifactId>nimbus-jose-jwt</artifactId>
+        <version>${nimbusds.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>commons-codec</groupId>
+        <artifactId>commons-codec</artifactId>
+        <version>${commonscodec.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>commons-collections</groupId>
+        <artifactId>commons-collections</artifactId>
+        <version>${commonscollections.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>commons-io</groupId>
+        <artifactId>commons-io</artifactId>
+        <version>${commonsio.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>commons-logging</groupId>
+        <artifactId>commons-logging</artifactId>
+        <version>${commonslogging.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-buffer</artifactId>
+        <version>${netty.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-common</artifactId>
+        <version>${netty.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>net.snowflake</groupId>
+        <artifactId>snowflake-jdbc</artifactId>
+        <version>${snowjdbc.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.commons</groupId>
+        <artifactId>commons-compress</artifactId>
+        <version>${commonscompress.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.commons</groupId>
+        <artifactId>commons-lang3</artifactId>
+        <version>${commonslang3.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-common</artifactId>
+        <version>${hadoop.version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>ch.qos.reload4j</groupId>
+            <artifactId>reload4j</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>javax.activation</groupId>
+            <artifactId>activation</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-reload4j</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.parquet</groupId>
+        <artifactId>parquet-column</artifactId>
+        <version>${parquet.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.parquet</groupId>
+        <artifactId>parquet-common</artifactId>
+        <version>${parquet.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.parquet</groupId>
+        <artifactId>parquet-hadoop</artifactId>
+        <version>${parquet.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.yetus</groupId>
+        <artifactId>audience-annotations</artifactId>
+        <version>${yetus.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.codehaus.jackson</groupId>
+        <artifactId>jackson-core-asl</artifactId>
+        <version>${codehaus.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.codehaus.jackson</groupId>
+        <artifactId>jackson-jaxrs</artifactId>
+        <version>${codehaus.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.codehaus.jackson</groupId>
+        <artifactId>jackson-mapper-asl</artifactId>
+        <version>${codehaus.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.codehaus.jackson</groupId>
+        <artifactId>jackson-xc</artifactId>
+        <version>${codehaus.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.codehaus.woodstox</groupId>
+        <artifactId>stax2-api</artifactId>
+        <version>4.2.1</version>
+      </dependency>
+      <dependency>
+        <groupId>org.objenesis</groupId>
+        <artifactId>objenesis</artifactId>
+        <version>${objenesis.version}</version>
+      </dependency>
+      <!-- SLF4J api that a client can shim in later -->
+      <dependency>
+        <groupId>org.slf4j</groupId>
+        <artifactId>slf4j-api</artifactId>
+        <version>${slf4j.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.xerial.snappy</groupId>
+        <artifactId>snappy-java</artifactId>
+        <version>${snappy.version}</version>
+      </dependency>
 
-        <!-- Snowflake JDBC used to connect to the service-->
-        <dependency>
-            <groupId>net.snowflake</groupId>
-            <artifactId>snowflake-jdbc</artifactId>
-            <version>3.13.15</version>
-        </dependency>
+      <!-- JUnit so that we can make some basic unit tests -->
+      <dependency>
+        <groupId>junit</groupId>
+        <artifactId>junit</artifactId>
+        <version>4.13.2</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>net.bytebuddy</groupId>
+        <artifactId>byte-buddy</artifactId>
+        <version>1.10.19</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>net.bytebuddy</groupId>
+        <artifactId>byte-buddy-agent</artifactId>
+        <version>1.10.19</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.mockito</groupId>
+        <artifactId>mockito-core</artifactId>
+        <version>3.7.7</version>
+        <scope>test</scope>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
 
-        <!-- jwt token for key pair authentication with GS -->
-        <dependency>
-            <groupId>com.nimbusds</groupId>
-            <artifactId>nimbus-jose-jwt</artifactId>
-            <version>9.9.3</version>
-         </dependency>
+  <!-- All of our needed dependencies -->
+  <dependencies>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-annotations</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
+    </dependency>
+    <!-- Jackson Databind api -->
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+    </dependency>
 
-        <!-- Jackson Databind api -->
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>2.14.0</version>
-        </dependency>
+    <dependency>
+      <groupId>com.google.code.findbugs</groupId>
+      <artifactId>jsr305</artifactId>
+      <version>3.0.2</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
 
-        <!-- SLF4J api that a client can shim in later -->
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.21</version>
-            <scope>provided</scope>
-        </dependency>
+    <!-- jwt token for key pair authentication with GS -->
+    <dependency>
+      <groupId>com.nimbusds</groupId>
+      <artifactId>nimbus-jose-jwt</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>commons-codec</groupId>
+      <artifactId>commons-codec</artifactId>
+    </dependency>
 
-        <!-- JDK logger backend for logging tests -->
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-simple</artifactId>
-            <version>1.7.21</version>
-            <scope>test</scope>
-        </dependency>
+    <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-core -->
+    <dependency>
+      <groupId>io.dropwizard.metrics</groupId>
+      <artifactId>metrics-core</artifactId>
+      <version>4.1.22</version>
+    </dependency>
 
-        <!-- JUnit so that we can make some basic unit tests -->
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.13.2</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.powermock</groupId>
-            <artifactId>powermock-module-junit4</artifactId>
-            <version>2.0.2</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>3.7.7</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.powermock</groupId>
-            <artifactId>powermock-api-mockito2</artifactId>
-            <version>2.0.2</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.powermock</groupId>
-            <artifactId>powermock-core</artifactId>
-            <version>2.0.2</version>
-            <scope>test</scope>
-        </dependency>
+    <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-jmx -->
+    <dependency>
+      <groupId>io.dropwizard.metrics</groupId>
+      <artifactId>metrics-jmx</artifactId>
+      <version>4.1.22</version>
+    </dependency>
 
-        <!-- Apache Arrow -->
-        <dependency>
-            <groupId>org.apache.arrow</groupId>
-            <artifactId>arrow-vector</artifactId>
-            <version>${arrow.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.arrow</groupId>
-            <artifactId>arrow-memory-netty</artifactId>
-            <version>${arrow.version}</version>
-            <scope>runtime</scope>
-        </dependency>
+    <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-jvm -->
+    <dependency>
+      <groupId>io.dropwizard.metrics</groupId>
+      <artifactId>metrics-jvm</artifactId>
+      <version>4.1.22</version>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>net.bytebuddy</groupId>
+      <artifactId>byte-buddy-agent</artifactId>
+    </dependency>
+    <!-- Snowflake JDBC used to connect to the service-->
+    <dependency>
+      <groupId>net.snowflake</groupId>
+      <artifactId>snowflake-jdbc</artifactId>
+    </dependency>
+    <!-- Apache Arrow -->
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-memory-core</artifactId>
+      <version>${arrow.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-vector</artifactId>
+      <version>${arrow.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+    </dependency>
+    <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop -->
+    <dependency>
+      <groupId>org.apache.parquet</groupId>
+      <artifactId>parquet-column</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.parquet</groupId>
+      <artifactId>parquet-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.parquet</groupId>
+      <artifactId>parquet-hadoop</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-memory-netty</artifactId>
+      <version>${arrow.version}</version>
+      <scope>runtime</scope>
+    </dependency>
+    <!-- JUnit so that we can make some basic unit tests -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.hamcrest</groupId>
+      <artifactId>hamcrest-core</artifactId>
+      <version>1.3</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.powermock</groupId>
+      <artifactId>powermock-api-mockito2</artifactId>
+      <version>2.0.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.powermock</groupId>
+      <artifactId>powermock-core</artifactId>
+      <version>2.0.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.powermock</groupId>
+      <artifactId>powermock-module-junit4</artifactId>
+      <version>2.0.2</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
 
-        <!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-hadoop -->
-        <dependency>
-            <groupId>org.apache.parquet</groupId>
-            <artifactId>parquet-hadoop</artifactId>
-            <version>1.12.3</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>javax.annotation</groupId>
-                    <artifactId>javax.annotation-api</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
+  <build>
+    <finalName>${project.artifactId}</finalName>
+    <resources>
+      <resource>
+        <filtering>true</filtering>
+        <directory>src/main/resources</directory>
+      </resource>
+    </resources>
 
-        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-mapreduce-client-core</artifactId>
-            <version>${hadoop.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.apache.hadoop</groupId>
-                    <artifactId>hadoop-yarn-common</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.hadoop</groupId>
-                    <artifactId>hadoop-hdfs-client</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.google.protobuf</groupId>
-                    <artifactId>protobuf-java</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.avro</groupId>
-                    <artifactId>avro</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.google.inject.extensions</groupId>
-                    <artifactId>guice-servlet</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.eclipse.jetty.websocket</groupId>
-                    <artifactId>websocket-client</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>io.netty</groupId>
-                    <artifactId>netty</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-reload4j</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-common</artifactId>
-            <version>${hadoop.version}</version>
+    <!-- disable default maven deploy plugin since we are using gpg:sign-and-deploy-file -->
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <artifactId>maven-deploy-plugin</artifactId>
+          <configuration>
+            <skip>true</skip>
+          </configuration>
+        </plugin>
+        <!-- Need to define the maven-failsafe-plugin here -->
+        <!-- http://maven.apache.org/surefire/maven-failsafe-plugin/plugin-info.html -->
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-failsafe-plugin</artifactId>
+          <version>3.0.0-M5</version>
+        </plugin>
+        <plugin>
+          <groupId>org.jacoco</groupId>
+          <artifactId>jacoco-maven-plugin</artifactId>
+          <version>${jacoco.version}</version>
+          <configuration>
+            <skip>${jacoco.skip.instrument}</skip>
+          </configuration>
+          <executions>
+            <execution>
+              <id>pre-unit-test</id>
+              <goals>
+                <goal>prepare-agent</goal>
+              </goals>
+              <configuration>
+                <destFile>target/jacoco-ut.exec</destFile>
+              </configuration>
+            </execution>
+            <execution>
+              <id>post-unit-test</id>
+              <goals>
+                <goal>report</goal>
+              </goals>
+              <phase>test</phase>
+              <configuration>
+                <dataFile>target/jacoco-ut.exec</dataFile>
+                <outputDirectory>target/jacoco-ut</outputDirectory>
+              </configuration>
+            </execution>
+          </executions>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+    <plugins>
+      <plugin>
+        <groupId>com.github.ekryd.sortpom</groupId>
+        <artifactId>sortpom-maven-plugin</artifactId>
+        <version>3.0.1</version>
+        <configuration>
+          <createBackupFile>false</createBackupFile>
+          <expandEmptyElements>false</expandEmptyElements>
+          <indentSchemaLocation>true</indentSchemaLocation>
+          <sortDependencies>scope,groupId,artifactId</sortDependencies>
+          <sortDependencyExclusions>groupId,artifactId</sortDependencyExclusions>
+          <sortExecutions>true</sortExecutions>
+          <sortModules>true</sortModules>
+          <sortPlugins>groupId,artifactId</sortPlugins>
+          <sortProperties>true</sortProperties>
+          <verifyFail>stop</verifyFail>
+          <verifyFailOn>strict</verifyFailOn>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>verify</goal>
+            </goals>
+            <phase>validate</phase>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <configuration>
+          <archive>
+            <manifest>
+              <addClasspath>true</addClasspath>
+              <mainClass>net.snowflake.ingest.SimpleIngestManager</mainClass>
+            </manifest>
+          </archive>
+          <descriptorRefs>
+            <descriptorRef>jar-with-dependencies</descriptorRef>
+          </descriptorRefs>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>2.5.1</version>
+        <inherited>true</inherited>
+        <configuration>
+          <source>1.8</source>
+          <target>1.8</target>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <version>3.3.0</version>
+        <executions>
+          <execution>
+            <id>analyze</id>
+            <goals>
+              <goal>analyze-only</goal>
+            </goals>
+            <configuration>
+              <failOnWarning>true</failOnWarning>
+              <ignoreNonCompile>true</ignoreNonCompile>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <version>3.0.0-M3</version>
+        <dependencies>
+          <dependency>
+            <groupId>com.google.cloud.tools</groupId>
+            <artifactId>linkage-checker-enforcer-rules</artifactId>
+            <version>1.5.12</version>
+          </dependency>
+          <dependency>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>extra-enforcer-rules</artifactId>
+            <version>1.3</version>
             <exclusions>
-                <exclusion>
-                    <groupId>com.github.spotbugs</groupId>
-                    <artifactId>spotbugs-annotations</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.nimbusds</groupId>
-                    <artifactId>nimbus-jose-jwt</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.sun.jersey</groupId>
-                    <artifactId>jersey-core</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.sun.jersey</groupId>
-                    <artifactId>jersey-json</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.sun.jersey</groupId>
-                    <artifactId>jersey-server</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.sun.jersey</groupId>
-                    <artifactId>jersey-servlet</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>javax.servlet</groupId>
-                    <artifactId>servlet-api</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>javax.servlet</groupId>
-                    <artifactId>javax.servlet-api</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>javax.servlet.jsp</groupId>
-                    <artifactId>jsp-api</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>log4j</groupId>
-                    <artifactId>log4j</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.codehaus.jackson</groupId>
-                    <artifactId>jackson-mapper-asl</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.google.protobuf</groupId>
-                    <artifactId>protobuf-java</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.mortbay.jetty</groupId>
-                    <artifactId>jetty</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.eclipse.jetty</groupId>
-                    <artifactId>jetty-server</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-log4j12</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.zookeeper</groupId>
-                    <artifactId>zookeeper</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.avro</groupId>
-                    <artifactId>avro</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.slf4j</groupId>
-                    <artifactId>slf4j-reload4j</artifactId>
-                </exclusion>
+              <exclusion>
+                <groupId>org.eclipse.aether</groupId>
+                <artifactId>aether-util</artifactId>
+              </exclusion>
             </exclusions>
-        </dependency>
+          </dependency>
+        </dependencies>
+        <executions>
+          <execution>
+            <id>enforce-best-practices</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <configuration>
+              <rules>
+                <banDuplicateClasses>
+                  <findAllDuplicates>true</findAllDuplicates>
+                  <ignoreWhenIdentical>true</ignoreWhenIdentical>
+                </banDuplicateClasses>
+                <banDuplicatePomDependencyVersions/>
+                <bannedDependencies/>
+                <dependencyConvergence/>
+                <requireUpperBoundDeps/>
+              </rules>
+            </configuration>
+          </execution>
+          <execution>
+            <id>enforce-maven</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <configuration>
+              <rules>
+                <requireMavenVersion>
+                  <version>3.5</version>
+                </requireMavenVersion>
+              </rules>
+            </configuration>
+          </execution>
+          <execution>
+            <id>enforce-linkage-checker</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <phase>verify</phase>
+            <configuration>
+              <rules>
+                <LinkageCheckerRule implementation="com.google.cloud.tools.dependencies.enforcer.LinkageCheckerRule">
+                  <reportOnlyReachable>true</reportOnlyReachable>
+                  <exclusionFile>${basedir}/linkage-checker-exclusion-rules.xml</exclusionFile>
+                  <level>WARN</level>
+                </LinkageCheckerRule>
+                <bannedDependencies>
+                  <excludes>
+                    <!-- JUnit 4 is not allowed on the classpath to prevent its accidental use -->
+                    <!-- <exclude>junit:junit</exclude> -->
+                  </excludes>
+                </bannedDependencies>
+              </rules>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <version>2.10.4</version>
+        <executions>
+          <execution>
+            <id>attach-javadocs</id>
+            <goals>
+              <goal>jar</goal>
+            </goals>
+            <configuration>
+              <additionalparam>-Xdoclint:none</additionalparam>
+              <source>8</source>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-source-plugin</artifactId>
+        <version>3.0.1</version>
+        <executions>
+          <execution>
+            <id>attach-sources</id>
+            <goals>
+              <goal>jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>2.19.1</version>
+        <configuration>
+          <excludes>
+            <exclude>**/TestSimpleIngestLocal.java</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>check-shaded-content</id>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <phase>verify</phase>
+            <configuration>
+              <executable>${basedir}/scripts/check_content.sh</executable>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
 
-        <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-core -->
-        <dependency>
-            <groupId>io.dropwizard.metrics</groupId>
-            <artifactId>metrics-core</artifactId>
-            <version>4.1.22</version>
-        </dependency>
-
-        <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-jvm -->
-        <dependency>
-            <groupId>io.dropwizard.metrics</groupId>
-            <artifactId>metrics-jvm</artifactId>
-            <version>4.1.22</version>
-        </dependency>
-
-        <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-jmx -->
-        <dependency>
-            <groupId>io.dropwizard.metrics</groupId>
-            <artifactId>metrics-jmx</artifactId>
-            <version>4.2.3</version>
-        </dependency>
-    </dependencies>
-
-    <profiles>
-        <profile>
-            <id>shadeDep</id>
-            <activation>
-                <property>
-                    <name>!not-shadeDep</name>
-                </property>
-            </activation>
-            <build>
-                <plugins>
-                    <!-- Relocate all dependencies to internal to solve dependency conflict problem -->
-                    <plugin>
-                        <groupId>org.apache.maven.plugins</groupId>
-                        <artifactId>maven-shade-plugin</artifactId>
-                        <version>3.1.1</version>
-                        <configuration>
-                            <relocations>
-                                <relocation>
-                                    <pattern>com.nimbusds</pattern>
-                                    <shadedPattern>
-                                        ${shadeBase}.com.nimbusds
-                                    </shadedPattern>
-                                </relocation>
-                                <relocation>
-                                    <pattern>net.jcip</pattern>
-                                    <shadedPattern>${shadeBase}.net.jcip</shadedPattern>
-                                </relocation>
-                                <relocation>
-                                    <pattern>net.minidev</pattern>
-                                    <shadedPattern>${shadeBase}.net.minidev</shadedPattern>
-                                </relocation>
-                                <relocation>
-                                    <pattern>org.objectweb</pattern>
-                                    <shadedPattern>${shadeBase}.org.objectweb</shadedPattern>
-                                </relocation>
-                                <relocation>
-                                    <pattern>com.fasterxml</pattern>
-                                    <shadedPattern>${shadeBase}.fasterxml</shadedPattern>
-                                </relocation>
-                                <relocation>
-                                    <pattern>org.apache</pattern>
-                                    <shadedPattern>${shadeBase}.apache</shadedPattern>
-                                </relocation>
-                            </relocations>
-                            <filters>
-                                <filter>
-                                    <artifact>*:*</artifact>
-                                    <excludes>
-                                        <exclude>META-INF/LICENSE*</exclude>
-                                        <exclude>META-INF/NOTICE*</exclude>
-                                        <exclude>META-INF/DEPENDENCIES</exclude>
-                                        <exclude>META-INF/maven/**</exclude>
-                                        <exclude>META-INF/services/com.fasterxml.*</exclude>
-                                        <exclude>META-INF/*.xml</exclude>
-                                        <exclude>META-INF/*.SF</exclude>
-                                        <exclude>META-INF/*.DSA</exclude>
-                                        <exclude>META-INF/*.RSA</exclude>
-                                    </excludes>
-                                </filter>
-                                <filter>
-                                    <artifact>commons-logging:commons-logging</artifact>
-                                    <excludes>
-                                        <exclude>org/apache/commons/logging/impl/AvalonLogger.class</exclude>
-                                    </excludes>
-                                </filter>
-                                <filter>
-                                    <artifact>org.slf4j:slf4j-simple</artifact>
-                                    <excludes>
-                                        <exclude>org/slf4j/**</exclude>
-                                    </excludes>
-                                </filter>
-                            </filters>
-                            <transformers>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                </transformer>
-                            </transformers>
-                        </configuration>
-                        <executions>
-                            <execution>
-                                <phase>package</phase>
-                                <goals>
-                                    <goal>shade</goal>
-                                </goals>
-                            </execution>
-                        </executions>
-                    </plugin>
-                </plugins>
-            </build>
-        </profile>
-        <profile>
-            <id>ossrh-deploy</id>
-            <activation>
-                <property>
-                    <name>ossrh-deploy</name>
-                </property>
-            </activation>
-            <build>
-                <plugins>
-                    <plugin>
-                        <groupId>org.apache.maven.plugins</groupId>
-                        <artifactId>maven-gpg-plugin</artifactId>
-                        <version>1.6</version>
-                        <executions>
-                            <execution>
-                                <phase>deploy</phase>
-                                <goals>
-                                    <goal>sign-and-deploy-file</goal>
-                                </goals>
-                                <configuration>
-                                    <file>target/${project.artifactId}.jar</file>
-                                    <repositoryId>ossrh</repositoryId>
-                                    <url>https://oss.sonatype.org/service/local/staging/deploy/maven2</url>
-                                    <pomFile>generated_public_pom.xml</pomFile>
-                                    <javadoc>target/${project.artifactId}-javadoc.jar</javadoc>
-                                    <sources>target/${project.artifactId}-sources.jar</sources>
-                                    <keyname>${env.GPG_KEY_ID}</keyname>
-                                    <passphrase>${env.GPG_KEY_PASSPHRASE}</passphrase>
-                                </configuration>
-                            </execution>
-                        </executions>
-                    </plugin>
-                </plugins>
-            </build>
-        </profile>
-        <profile>
-            <id>ghActionsIT</id>
-            <activation>
-                <property>
-                    <name>ghActionsIT</name>
-                </property>
-            </activation>
-            <build>
-                <plugins>
-                    <plugin>
-                        <groupId>org.apache.maven.plugins</groupId>
-                        <artifactId>maven-failsafe-plugin</artifactId>
-                        <executions>
-                            <execution>
-                                <goals>
-                                    <goal>integration-test</goal>
-                                </goals>
-                            </execution>
-                            <execution>
-                                <id>verify_github_actions_it</id>
-                                <phase>verify</phase>
-                                <goals>
-                                    <goal>verify</goal>
-                                </goals>
-                            </execution>
-                        </executions>
-                    </plugin>
-                    <plugin>
-                        <groupId>org.jacoco</groupId>
-                        <artifactId>jacoco-maven-plugin</artifactId>
-                        <version>${jacoco.version}</version>
-                        <executions>
-                            <execution>
-                                <id>pre-unit-test</id>
-                                <goals>
-                                    <goal>prepare-agent</goal>
-                                </goals>
-                                <configuration>
-                                    <destFile>target/jacoco-ut.exec</destFile>
-                                </configuration>
-                            </execution>
-                            <execution>
-                                <id>post-unit-test</id>
-                                <phase>test</phase>
-                                <goals>
-                                    <goal>report</goal>
-                                </goals>
-                                <configuration>
-                                    <dataFile>target/jacoco-ut.exec</dataFile>
-                                    <outputDirectory>target/jacoco-ut</outputDirectory>
-                                </configuration>
-                            </execution>
-                            <execution>
-                                <id>pre-integration-test</id>
-                                <phase>pre-integration-test</phase>
-                                <goals>
-                                    <goal>prepare-agent</goal>
-                                </goals>
-                                <configuration>
-                                    <destFile>target/jacoco-it.exec</destFile>
-                                </configuration>
-                            </execution>
-                            <execution>
-                                <id>post-integration-test</id>
-                                <phase>post-integration-test</phase>
-                                <goals>
-                                    <goal>report</goal>
-                                </goals>
-                                <configuration>
-                                    <dataFile>target/jacoco-it.exec</dataFile>
-                                    <outputDirectory>target/jacoco-it</outputDirectory>
-                                </configuration>
-                            </execution>
-                        </executions>
-                        <configuration>
-                            <skip>${jacoco.skip.instrument}</skip>
-                        </configuration>
-                    </plugin>
-                </plugins>
-            </build>
-        </profile>
-    </profiles>
+  <profiles>
+    <profile>
+      <id>shadeDep</id>
+      <activation>
+        <property>
+          <name>!not-shadeDep</name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <!-- Relocate all dependencies to internal to solve dependency conflict problem -->
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-shade-plugin</artifactId>
+            <version>3.1.1</version>
+            <configuration>
+              <relocations>
+                <relocation>
+                  <pattern>com.nimbusds</pattern>
+                  <shadedPattern>${shadeBase}.com.nimbusds</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>net.jcip</pattern>
+                  <shadedPattern>${shadeBase}.net.jcip</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>net.minidev</pattern>
+                  <shadedPattern>${shadeBase}.net.minidev</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.objectweb</pattern>
+                  <shadedPattern>${shadeBase}.org.objectweb</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>com.fasterxml</pattern>
+                  <shadedPattern>${shadeBase}.fasterxml</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.apache</pattern>
+                  <shadedPattern>${shadeBase}.apache</shadedPattern>
+                </relocation>
+              </relocations>
+              <filters>
+                <filter>
+                  <artifact>*:*</artifact>
+                  <excludes>
+                    <exclude>META-INF/LICENSE*</exclude>
+                    <exclude>META-INF/NOTICE*</exclude>
+                    <exclude>META-INF/DEPENDENCIES</exclude>
+                    <exclude>META-INF/maven/**</exclude>
+                    <exclude>META-INF/services/com.fasterxml.*</exclude>
+                    <exclude>META-INF/*.xml</exclude>
+                    <exclude>META-INF/*.SF</exclude>
+                    <exclude>META-INF/*.DSA</exclude>
+                    <exclude>META-INF/*.RSA</exclude>
+                  </excludes>
+                </filter>
+                <filter>
+                  <artifact>commons-logging:commons-logging</artifact>
+                  <excludes>
+                    <exclude>org/apache/commons/logging/impl/AvalonLogger.class</exclude>
+                  </excludes>
+                </filter>
+                <filter>
+                  <artifact>org.slf4j:slf4j-simple</artifact>
+                  <excludes>
+                    <exclude>org/slf4j/**</exclude>
+                  </excludes>
+                </filter>
+              </filters>
+              <transformers>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"/>
+              </transformers>
+            </configuration>
+            <executions>
+              <execution>
+                <goals>
+                  <goal>shade</goal>
+                </goals>
+                <phase>package</phase>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <id>ossrh-deploy</id>
+      <activation>
+        <property>
+          <name>ossrh-deploy</name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-gpg-plugin</artifactId>
+            <version>1.6</version>
+            <executions>
+              <execution>
+                <goals>
+                  <goal>sign-and-deploy-file</goal>
+                </goals>
+                <phase>deploy</phase>
+                <configuration>
+                  <file>target/${project.artifactId}.jar</file>
+                  <repositoryId>ossrh</repositoryId>
+                  <url>https://oss.sonatype.org/service/local/staging/deploy/maven2</url>
+                  <pomFile>generated_public_pom.xml</pomFile>
+                  <javadoc>target/${project.artifactId}-javadoc.jar</javadoc>
+                  <sources>target/${project.artifactId}-sources.jar</sources>
+                  <keyname>${env.GPG_KEY_ID}</keyname>
+                  <passphrase>${env.GPG_KEY_PASSPHRASE}</passphrase>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <id>ghActionsIT</id>
+      <activation>
+        <property>
+          <name>ghActionsIT</name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-failsafe-plugin</artifactId>
+            <executions>
+              <execution>
+                <goals>
+                  <goal>integration-test</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>verify_github_actions_it</id>
+                <goals>
+                  <goal>verify</goal>
+                </goals>
+                <phase>verify</phase>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <groupId>org.jacoco</groupId>
+            <artifactId>jacoco-maven-plugin</artifactId>
+            <version>${jacoco.version}</version>
+            <configuration>
+              <skip>${jacoco.skip.instrument}</skip>
+            </configuration>
+            <executions>
+              <execution>
+                <id>pre-unit-test</id>
+                <goals>
+                  <goal>prepare-agent</goal>
+                </goals>
+                <configuration>
+                  <destFile>target/jacoco-ut.exec</destFile>
+                </configuration>
+              </execution>
+              <execution>
+                <id>post-unit-test</id>
+                <goals>
+                  <goal>report</goal>
+                </goals>
+                <phase>test</phase>
+                <configuration>
+                  <dataFile>target/jacoco-ut.exec</dataFile>
+                  <outputDirectory>target/jacoco-ut</outputDirectory>
+                </configuration>
+              </execution>
+              <execution>
+                <id>pre-integration-test</id>
+                <goals>
+                  <goal>prepare-agent</goal>
+                </goals>
+                <phase>pre-integration-test</phase>
+                <configuration>
+                  <destFile>target/jacoco-it.exec</destFile>
+                </configuration>
+              </execution>
+              <execution>
+                <id>post-integration-test</id>
+                <goals>
+                  <goal>report</goal>
+                </goals>
+                <phase>post-integration-test</phase>
+                <configuration>
+                  <dataFile>target/jacoco-it.exec</dataFile>
+                  <outputDirectory>target/jacoco-it</outputDirectory>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
 
 </project>

From 56b6a74a3eebaf35a52a6e9f1223bc5fa9311a41 Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Tue, 28 Mar 2023 11:10:05 +0200
Subject: [PATCH 149/356] SNOW-759237 edit few log lines and comments (#364)

---
 .../streaming/internal/AbstractRowBuffer.java |  10 +-
 .../streaming/internal/BlobBuilder.java       |  13 +-
 .../streaming/internal/ChannelData.java       |   3 +-
 .../ingest/streaming/internal/RowBuffer.java  |   2 +-
 ...nowflakeStreamingIngestClientInternal.java |   2 +-
 .../ingest/ingest_error_messages.properties   |   2 +-
 .../internal/DataValidationUtilTest.java      | 152 +++++++++---------
 .../datatypes/AbstractDataTypeTest.java       |   3 +-
 8 files changed, 97 insertions(+), 90 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 0a8e3087d..fa57199db 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -32,7 +32,8 @@
  * un-flushed rows, these rows will be converted to the underlying format implementation for faster
  * processing
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot})
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or Parquet
+ *     {@link ParquetChunkData})
  */
 abstract class AbstractRowBuffer<T> implements RowBuffer<T> {
   private static final Logging logger = new Logging(AbstractRowBuffer.class);
@@ -379,8 +380,7 @@ public ChannelData<T> flush(final String filePath) {
       Map<String, RowBufferStats> oldColumnEps = null;
       Pair<Long, Long> oldMinMaxInsertTimeInMs = null;
 
-      logger.logDebug(
-          "Arrow buffer flush about to take lock on channel={}", channelFullyQualifiedName);
+      logger.logDebug("Buffer flush about to take lock on channel={}", channelFullyQualifiedName);
 
       this.flushLock.lock();
       try {
@@ -403,7 +403,7 @@ public ChannelData<T> flush(final String filePath) {
       }
 
       logger.logDebug(
-          "Arrow buffer flush released lock on channel={}, rowCount={}, bufferSize={}",
+          "Buffer flush released lock on channel={}, rowCount={}, bufferSize={}",
           channelFullyQualifiedName,
           oldRowCount,
           oldBufferSize);
@@ -527,7 +527,7 @@ public synchronized void close(String name) {
   /**
    * Given a set of col names to stats, build the right ep info
    *
-   * @param rowCount: count of rows in the given arrow buffer
+   * @param rowCount: count of rows in the given buffer
    * @param colStats: map of column name to RowBufferStats
    * @return the EPs built from column stats
    */
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
index 4249130bb..235f6c814 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
@@ -48,8 +48,8 @@
  *   <li>variable size chunks metadata in json format
  * </ul>
  *
- * <p>After the metadata, it will be one or more chunks of variable size Arrow data, and each chunk
- * will be encrypted and compressed separately.
+ * <p>After the metadata, it will be one or more chunks of variable size Arrow/Parquet data, and
+ * each chunk will be encrypted and compressed separately.
  */
 class BlobBuilder {
 
@@ -209,7 +209,8 @@ static Pair<byte[], Integer> compress(
 
   /**
    * Gzip compress the given chunk data if required by the given write mode and pads the compressed
-   * data for encryption.
+   * data for encryption. Only for Arrow. For Parquet the compression is done in the Parquet
+   * library.
    *
    * @param filePath blob file full path
    * @param chunkData uncompressed chunk data
@@ -280,11 +281,11 @@ static byte[] buildBlob(
       blob.write(toByteArray(chunkMetadataListInBytes.length));
       blob.write(chunkMetadataListInBytes);
     }
-    for (byte[] arrowData : chunksDataList) {
-      blob.write(arrowData);
+    for (byte[] chunkData : chunksDataList) {
+      blob.write(chunkData);
     }
 
-    // We need to update the start offset for the EP and Arrow data in the request since
+    // We need to update the start offset for the EP and Arrow/Parquet data in the request since
     // some metadata was added at the beginning
     for (ChunkMetadata chunkMetadata : chunksMetadataList) {
       chunkMetadata.advanceStartOffset(metadataSize);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
index 91f2a2a68..07ce4b989 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
@@ -15,7 +15,8 @@
  * Contains the data and metadata returned for each channel flush, which will be used to build the
  * blob and register blob request
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot}
+ * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or Parquet
+ *     {@link ParquetChunkData}
  */
 class ChannelData<T> {
   private Long rowSequencer;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
index 68c225cd4..5b7e937cd 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
@@ -17,7 +17,7 @@
  */
 interface RowBuffer<T> {
   /**
-   * Setup the column fields and vectors using the column metadata from the server
+   * Set up the column fields and vectors using the column metadata from the server
    *
    * @param columns list of column metadata
    */
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 77e25aa76..10d4d2377 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -132,7 +132,7 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
   Timer uploadLatency; // Latency for uploading a blob
   Timer registerLatency; // Latency for registering a blob
   Meter uploadThroughput; // Throughput for uploading blobs
-  Meter inputThroughput; // Throughput for inserting into the Arrow buffer
+  Meter inputThroughput; // Throughput for inserting into the internal buffer
 
   // JVM and thread related metrics
   MetricRegistry jvmMemoryAndThreadMetrics;
diff --git a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
index 807486bfc..49ac31d92 100644
--- a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
+++ b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
@@ -6,7 +6,7 @@
 0001=Ingest client internal error: {0}.
 0002=Required value is null, Key: {0}.
 0003=Required value is empty, Key: {0}.
-0004=The given row cannot be converted to Arrow format: {0}. {1}
+0004=The given row cannot be converted to the internal format: {0}. {1}
 0005=Unknown data type for logical: {0}, physical: {1}.
 0006=Register blob request failed: {0}.
 0007=Open channel request failed: {0}.
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 55e3298d9..9c3279c41 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -586,21 +586,21 @@ public void testTooLargeMultiByteSemiStructuredValues() {
     m.put("a", new String(stringContent));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: {\"a\":\"ČČČČČČČČČČČČČČ.... Value"
-            + " cannot be ingested into Snowflake column COL of type VARIANT: Variant too long:"
-            + " length=18874376 maxLength=16777152",
+        "The given row cannot be converted to the internal format: {\"a\":\"ČČČČČČČČČČČČČČ...."
+            + " Value cannot be ingested into Snowflake column COL of type VARIANT: Variant too"
+            + " long: length=18874376 maxLength=16777152",
         () -> validateAndParseVariant("COL", m));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: [{\"a\":\"ČČČČČČČČČČČČČ.... Value"
-            + " cannot be ingested into Snowflake column COL of type ARRAY: Array too large."
+        "The given row cannot be converted to the internal format: [{\"a\":\"ČČČČČČČČČČČČČ...."
+            + " Value cannot be ingested into Snowflake column COL of type ARRAY: Array too large."
             + " length=18874378 maxLength=16777152",
         () -> validateAndParseArray("COL", m));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: {\"a\":\"ČČČČČČČČČČČČČČ.... Value"
-            + " cannot be ingested into Snowflake column COL of type OBJECT: Object too large."
-            + " length=18874376 maxLength=16777152",
+        "The given row cannot be converted to the internal format: {\"a\":\"ČČČČČČČČČČČČČČ...."
+            + " Value cannot be ingested into Snowflake column COL of type OBJECT: Object too"
+            + " large. length=18874376 maxLength=16777152",
         () -> validateAndParseObject("COL", m));
   }
 
@@ -894,14 +894,14 @@ public void testExceptionMessages() {
     // BOOLEAN
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column COL of type BOOLEAN. Allowed Java types: boolean,"
-            + " Number, String",
+        "The given row cannot be converted to the internal format: Object of type java.lang.Object"
+            + " cannot be ingested into Snowflake column COL of type BOOLEAN. Allowed Java types:"
+            + " boolean, Number, String",
         () -> validateAndParseBoolean("COL", new Object()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column COL of type BOOLEAN: Not a valid boolean, see"
+        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
+            + " into Snowflake column COL of type BOOLEAN: Not a valid boolean, see"
             + " https://docs.snowflake.com/en/sql-reference/data-types-logical.html#conversion-to-boolean"
             + " for the list of supported formats",
         () -> validateAndParseBoolean("COL", "abc"));
@@ -909,29 +909,29 @@ public void testExceptionMessages() {
     // TIME
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column COL of type TIME. Allowed Java types: String,"
-            + " LocalTime, OffsetTime",
+        "The given row cannot be converted to the internal format: Object of type java.lang.Object"
+            + " cannot be ingested into Snowflake column COL of type TIME. Allowed Java types:"
+            + " String, LocalTime, OffsetTime",
         () -> validateAndParseTime("COL", new Object(), 10));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column COL of type TIME: Not a valid time, see"
-            + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview"
-            + " for the list of supported formats",
+        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
+            + " into Snowflake column COL of type TIME: Not a valid time, see"
+            + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
+            + " the list of supported formats",
         () -> validateAndParseTime("COL", "abc", 10));
 
     // DATE
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column COL of type DATE. Allowed Java types: String,"
-            + " LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
+        "The given row cannot be converted to the internal format: Object of type java.lang.Object"
+            + " cannot be ingested into Snowflake column COL of type DATE. Allowed Java types:"
+            + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
         () -> validateAndParseDate("COL", new Object()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column COL of type DATE: Not a valid value, see"
+        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
+            + " into Snowflake column COL of type DATE: Not a valid value, see"
             + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
             + " the list of supported formats",
         () -> validateAndParseDate("COL", "abc"));
@@ -939,14 +939,14 @@ public void testExceptionMessages() {
     // TIMESTAMP_NTZ
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
+        "The given row cannot be converted to the internal format: Object of type java.lang.Object"
+            + " cannot be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
             + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
         () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, true));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column COL of type TIMESTAMP: Not a valid value, see"
+        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
+            + " into Snowflake column COL of type TIMESTAMP: Not a valid value, see"
             + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
             + " the list of supported formats",
         () -> validateAndParseTimestamp("COL", "abc", 3, UTC, true));
@@ -954,14 +954,14 @@ public void testExceptionMessages() {
     // TIMESTAMP_LTZ
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
+        "The given row cannot be converted to the internal format: Object of type java.lang.Object"
+            + " cannot be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
             + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
         () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, false));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column COL of type TIMESTAMP: Not a valid value, see"
+        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
+            + " into Snowflake column COL of type TIMESTAMP: Not a valid value, see"
             + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
             + " the list of supported formats",
         () -> validateAndParseTimestamp("COL", "abc", 3, UTC, false));
@@ -969,14 +969,14 @@ public void testExceptionMessages() {
     // TIMESTAMP_TZ
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
+        "The given row cannot be converted to the internal format: Object of type java.lang.Object"
+            + " cannot be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
             + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
         () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, false));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column COL of type TIMESTAMP: Not a valid value, see"
+        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
+            + " into Snowflake column COL of type TIMESTAMP: Not a valid value, see"
             + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
             + " the list of supported formats",
         () -> validateAndParseTimestamp("COL", "abc", 3, UTC, false));
@@ -984,98 +984,102 @@ public void testExceptionMessages() {
     // NUMBER
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column COL of type NUMBER. Allowed Java types: int,"
-            + " long, byte, short, float, double, BigDecimal, BigInteger, String",
+        "The given row cannot be converted to the internal format: Object of type java.lang.Object"
+            + " cannot be ingested into Snowflake column COL of type NUMBER. Allowed Java types:"
+            + " int, long, byte, short, float, double, BigDecimal, BigInteger, String",
         () -> validateAndParseBigDecimal("COL", new Object()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column COL of type NUMBER: Not a valid number",
+        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
+            + " into Snowflake column COL of type NUMBER: Not a valid number",
         () -> validateAndParseBigDecimal("COL", "abc"));
 
     // REAL
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column COL of type REAL. Allowed Java types: Number,"
-            + " String",
+        "The given row cannot be converted to the internal format: Object of type java.lang.Object"
+            + " cannot be ingested into Snowflake column COL of type REAL. Allowed Java types:"
+            + " Number, String",
         () -> validateAndParseReal("COL", new Object()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column COL of type REAL: Not a valid decimal number",
+        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
+            + " into Snowflake column COL of type REAL: Not a valid decimal number",
         () -> validateAndParseReal("COL", "abc"));
 
     // STRING
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column COL of type STRING. Allowed Java types: String,"
-            + " Number, boolean, char",
+        "The given row cannot be converted to the internal format: Object of type java.lang.Object"
+            + " cannot be ingested into Snowflake column COL of type STRING. Allowed Java types:"
+            + " String, Number, boolean, char",
         () -> validateAndParseString("COL", new Object(), Optional.empty()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: abc. Value cannot be ingested into"
-            + " Snowflake column COL of type STRING: String too long: length=3 characters"
+        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
+            + " into Snowflake column COL of type STRING: String too long: length=3 characters"
             + " maxLength=2 characters",
         () -> validateAndParseString("COL", "abc", Optional.of(2)));
 
     // BINARY
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column COL of type BINARY. Allowed Java types: byte[],"
-            + " String",
+        "The given row cannot be converted to the internal format: Object of type java.lang.Object"
+            + " cannot be ingested into Snowflake column COL of type BINARY. Allowed Java types:"
+            + " byte[], String",
         () -> validateAndParseBinary("COL", new Object(), Optional.empty()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: byte[2]. Value cannot be ingested into"
-            + " Snowflake column COL of type BINARY: Binary too long: length=2 maxLength=1",
+        "The given row cannot be converted to the internal format: byte[2]. Value cannot be"
+            + " ingested into Snowflake column COL of type BINARY: Binary too long: length=2"
+            + " maxLength=1",
         () -> validateAndParseBinary("COL", new byte[] {1, 2}, Optional.of(1)));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: ghi. Value cannot be ingested into"
-            + " Snowflake column COL of type BINARY: Not a valid hex string",
+        "The given row cannot be converted to the internal format: ghi. Value cannot be ingested"
+            + " into Snowflake column COL of type BINARY: Not a valid hex string",
         () -> validateAndParseBinary("COL", "ghi", Optional.empty()));
 
     // VARIANT
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column COL of type VARIANT. Allowed Java types: String,"
-            + " Primitive data types and their arrays, java.time.*, List<T>, Map<String, T>, T[]",
+        "The given row cannot be converted to the internal format: Object of type java.lang.Object"
+            + " cannot be ingested into Snowflake column COL of type VARIANT. Allowed Java types:"
+            + " String, Primitive data types and their arrays, java.time.*, List<T>, Map<String,"
+            + " T>, T[]",
         () -> validateAndParseVariant("COL", new Object()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: ][. Value cannot be ingested into"
-            + " Snowflake column COL of type VARIANT: Not a valid JSON",
+        "The given row cannot be converted to the internal format: ][. Value cannot be ingested"
+            + " into Snowflake column COL of type VARIANT: Not a valid JSON",
         () -> validateAndParseVariant("COL", "]["));
 
     // ARRAY
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column COL of type ARRAY. Allowed Java types: String,"
-            + " Primitive data types and their arrays, java.time.*, List<T>, Map<String, T>, T[]",
+        "The given row cannot be converted to the internal format: Object of type java.lang.Object"
+            + " cannot be ingested into Snowflake column COL of type ARRAY. Allowed Java types:"
+            + " String, Primitive data types and their arrays, java.time.*, List<T>, Map<String,"
+            + " T>, T[]",
         () -> validateAndParseArray("COL", new Object()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: ][. Value cannot be ingested into"
-            + " Snowflake column COL of type ARRAY: Not a valid JSON",
+        "The given row cannot be converted to the internal format: ][. Value cannot be ingested"
+            + " into Snowflake column COL of type ARRAY: Not a valid JSON",
         () -> validateAndParseArray("COL", "]["));
 
     // OBJECT
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: Object of type java.lang.Object cannot"
-            + " be ingested into Snowflake column COL of type OBJECT. Allowed Java types: String,"
-            + " Primitive data types and their arrays, java.time.*, List<T>, Map<String, T>, T[]",
+        "The given row cannot be converted to the internal format: Object of type java.lang.Object"
+            + " cannot be ingested into Snowflake column COL of type OBJECT. Allowed Java types:"
+            + " String, Primitive data types and their arrays, java.time.*, List<T>, Map<String,"
+            + " T>, T[]",
         () -> validateAndParseObject("COL", new Object()));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to Arrow format: }{. Value cannot be ingested into"
-            + " Snowflake column COL of type OBJECT: Not a valid JSON",
+        "The given row cannot be converted to the internal format: }{. Value cannot be ingested"
+            + " into Snowflake column COL of type OBJECT: Not a valid JSON",
         () -> validateAndParseObject("COL", "}{"));
   }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index 49ab8b023..5ef3951c1 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -186,7 +186,8 @@ protected <T> void expectArrowNotSupported(String dataType, T value) throws Exce
         value,
         x ->
             (x instanceof SFException
-                && x.getMessage().contains("The given row cannot be converted to Arrow format")));
+                && x.getMessage()
+                    .contains("The given row cannot be converted to the internal format")));
   }
 
   /**

From 7b6e30b6778886b2a1ab276b51449b957994e64e Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Tue, 28 Mar 2023 16:05:57 +0200
Subject: [PATCH 150/356] SNOW-766066 Run tests on Windows (#369)

---
 .github/workflows/End2EndTest.yml             | 21 ++++++++-
 pom.xml                                       | 45 ++++++++++++-------
 .../internal/StreamingIngestStageTest.java    |  5 ++-
 3 files changed, 53 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/End2EndTest.yml b/.github/workflows/End2EndTest.yml
index d5307e6e8..5cd5d3fb9 100644
--- a/.github/workflows/End2EndTest.yml
+++ b/.github/workflows/End2EndTest.yml
@@ -35,4 +35,23 @@ jobs:
           ./scripts/run_gh_actions.sh
 
       - name: Code Coverage
-        uses: codecov/codecov-action@v1
\ No newline at end of file
+        uses: codecov/codecov-action@v1
+  build-windows:
+    runs-on: windows-2022
+    strategy:
+      matrix:
+        java: [ 8 ]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+      - name: Install Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+      - name: Decrypt profile.json (Windows)
+        env:
+          DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }}
+        run: gpg --quiet --batch --yes --decrypt --passphrase="${env:DECRYPTION_PASSPHRASE}" --output profile.json ./profile.json.gpg
+      - name: Unit & Integration Test (Windows)
+        continue-on-error: false
+        run: mvn -DghActionsIT verify --batch-mode
diff --git a/pom.xml b/pom.xml
index 884450447..2c1910e55 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,6 +54,7 @@
     <nimbusds.version>9.9.3</nimbusds.version>
     <objenesis.version>3.1</objenesis.version>
     <parquet.version>1.12.3</parquet.version>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <shadeBase>net.snowflake.ingest.internal</shadeBase>
     <slf4j.version>1.7.36</slf4j.version>
     <snappy.version>1.1.8.3</snappy.version>
@@ -630,26 +631,38 @@
           </excludes>
         </configuration>
       </plugin>
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>exec-maven-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>check-shaded-content</id>
-            <goals>
-              <goal>exec</goal>
-            </goals>
-            <phase>verify</phase>
-            <configuration>
-              <executable>${basedir}/scripts/check_content.sh</executable>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
     </plugins>
   </build>
 
   <profiles>
+    <profile>
+      <id>checkShadedContent</id>
+      <activation>
+        <os>
+          <family>!Windows</family>
+        </os>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>check-shaded-content</id>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <phase>verify</phase>
+                <configuration>
+                  <executable>${basedir}/scripts/check_content.sh</executable>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <profile>
       <id>shadeDep</id>
       <activation>
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
index 79c809e24..456154982 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
@@ -287,8 +287,11 @@ public void testRefreshSnowflakeMetadataRemote() throws Exception {
         StageInfo.StageType.S3, metadataWithAge.fileTransferMetadata.getStageInfo().getStageType());
     Assert.assertEquals(
         "foo/streaming_ingest/", metadataWithAge.fileTransferMetadata.getStageInfo().getLocation());
+    // Here we need to compare paths and not just strings because on windows, due to how JDBC driver
+    // works with path separators, presignedUrlFileName is absolute, but on Linux it is relative
     Assert.assertEquals(
-        "placeholder", metadataWithAge.fileTransferMetadata.getPresignedUrlFileName());
+        Paths.get("placeholder").toAbsolutePath(),
+        Paths.get(metadataWithAge.fileTransferMetadata.getPresignedUrlFileName()).toAbsolutePath());
     Assert.assertEquals(prefix + "_" + deploymentId, stage.getClientPrefix());
   }
 

From 13af83f912804069207ef6fda358c4ca9e6ca65a Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Tue, 28 Mar 2023 12:58:55 -0700
Subject: [PATCH 151/356] SNOW-744974: avoid combining channel data with
 different encryption key to the same chunk (#337)

We need this behavior in two cases:

- when there is a key rotation and it's possible that old channels and new channels are using different encryption key
- when the table is dropped and recreated with the same name, we would want to separate the data from old channels and new channels because the old channels data will be invalidated
---
 .../streaming/internal/FlushService.java      | 81 ++++++++++++++-----
 .../streaming/internal/FlushServiceTest.java  | 52 ++++++++++++
 2 files changed, 112 insertions(+), 21 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index b4b318a8c..7758e30e6 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -24,6 +24,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.TimeZone;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
@@ -34,7 +35,6 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
 import javax.crypto.BadPaddingException;
 import javax.crypto.IllegalBlockSizeException;
 import javax.crypto.NoSuchPaddingException;
@@ -338,33 +338,72 @@ void distributeFlushTasks() {
                 String, ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<T>>>>
         itr = this.channelCache.iterator();
     List<Pair<BlobData<T>, CompletableFuture<BlobMetadata>>> blobs = new ArrayList<>();
+    List<ChannelData<T>> leftoverChannelsDataPerTable = new ArrayList<>();
 
-    while (itr.hasNext()) {
+    while (itr.hasNext() || !leftoverChannelsDataPerTable.isEmpty()) {
       List<List<ChannelData<T>>> blobData = new ArrayList<>();
-      AtomicReference<Float> totalBufferSizeInBytes = new AtomicReference<>((float) 0);
-
+      float totalBufferSizeInBytes = 0F;
       final String filePath = getFilePath(this.targetStage.getClientPrefix());
 
-      // Distribute work at table level, create a new blob if reaching the blob size limit
-      while (itr.hasNext() && totalBufferSizeInBytes.get() <= MAX_BLOB_SIZE_IN_BYTES) {
-        ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<T>> table =
-            itr.next().getValue();
+      // Distribute work at table level, split the blob if reaching the blob size limit or the
+      // channel has different encryption key ids
+      while (itr.hasNext() || !leftoverChannelsDataPerTable.isEmpty()) {
         List<ChannelData<T>> channelsDataPerTable = Collections.synchronizedList(new ArrayList<>());
-        // Use parallel stream since getData could be the performance bottleneck when we have a high
-        // number of channels
-        table.values().parallelStream()
-            .forEach(
-                channel -> {
-                  if (channel.isValid()) {
-                    ChannelData<T> data = channel.getData(filePath);
-                    if (data != null) {
-                      channelsDataPerTable.add(data);
-                      totalBufferSizeInBytes.updateAndGet(v -> v + data.getBufferSize());
+        if (!leftoverChannelsDataPerTable.isEmpty()) {
+          channelsDataPerTable.addAll(leftoverChannelsDataPerTable);
+          leftoverChannelsDataPerTable.clear();
+        } else {
+          ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<T>> table =
+              itr.next().getValue();
+          // Use parallel stream since getData could be the performance bottleneck when we have a
+          // high number of channels
+          table.values().parallelStream()
+              .forEach(
+                  channel -> {
+                    if (channel.isValid()) {
+                      ChannelData<T> data = channel.getData(filePath);
+                      if (data != null) {
+                        channelsDataPerTable.add(data);
+                      }
                     }
-                  }
-                });
+                  });
+        }
+
         if (!channelsDataPerTable.isEmpty()) {
-          blobData.add(channelsDataPerTable);
+          int idx = 0;
+          while (idx < channelsDataPerTable.size()) {
+            ChannelData<T> channelData = channelsDataPerTable.get(idx);
+            // Stop processing the rest of channels if reaching the blob size limit or the channel
+            // has different encryption key ids
+            if (idx > 0
+                && (totalBufferSizeInBytes + channelData.getBufferSize() > MAX_BLOB_SIZE_IN_BYTES
+                    || !Objects.equals(
+                        channelData.getChannelContext().getEncryptionKeyId(),
+                        channelsDataPerTable
+                            .get(idx - 1)
+                            .getChannelContext()
+                            .getEncryptionKeyId()))) {
+              leftoverChannelsDataPerTable.addAll(
+                  channelsDataPerTable.subList(idx, channelsDataPerTable.size()));
+              logger.logInfo(
+                  "Creation of another blob is needed because of blob size limit or different"
+                      + " encryption ids, client={}, table={},  size={}, encryptionId1={},"
+                      + " encryptionId2={}",
+                  this.owningClient.getName(),
+                  channelData.getChannelContext().getTableName(),
+                  totalBufferSizeInBytes + channelData.getBufferSize(),
+                  channelData.getChannelContext().getEncryptionKeyId(),
+                  channelsDataPerTable.get(idx - 1).getChannelContext().getEncryptionKeyId());
+              break;
+            }
+            totalBufferSizeInBytes += channelData.getBufferSize();
+            idx++;
+          }
+          // Add processed channels to the current blob, stop if we need to create a new blob
+          blobData.add(channelsDataPerTable.subList(0, idx));
+          if (idx != channelsDataPerTable.size()) {
+            break;
+          }
         }
       }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 90344b8a6..0c485b94b 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -349,6 +349,8 @@ private SnowflakeStreamingIngestChannelInternal<?> addChannel1(TestContext<?> te
         .setOffsetToken("offset1")
         .setChannelSequencer(0L)
         .setRowSequencer(0L)
+        .setEncryptionKey("key")
+        .setEncryptionKeyId(1L)
         .buildAndAdd();
   }
 
@@ -361,6 +363,8 @@ private SnowflakeStreamingIngestChannelInternal<?> addChannel2(TestContext<?> te
         .setOffsetToken("offset2")
         .setChannelSequencer(10L)
         .setRowSequencer(100L)
+        .setEncryptionKey("key")
+        .setEncryptionKeyId(1L)
         .buildAndAdd();
   }
 
@@ -373,6 +377,22 @@ private SnowflakeStreamingIngestChannelInternal<?> addChannel3(TestContext<?> te
         .setOffsetToken("offset3")
         .setChannelSequencer(0L)
         .setRowSequencer(0L)
+        .setEncryptionKey("key3")
+        .setEncryptionKeyId(3L)
+        .buildAndAdd();
+  }
+
+  private SnowflakeStreamingIngestChannelInternal<?> addChannel4(TestContext<?> testContext) {
+    return testContext
+        .channelBuilder("channel4")
+        .setDBName("db1")
+        .setSchemaName("schema1")
+        .setTableName("table1")
+        .setOffsetToken("offset2")
+        .setChannelSequencer(10L)
+        .setRowSequencer(100L)
+        .setEncryptionKey("key4")
+        .setEncryptionKeyId(4L)
         .buildAndAdd();
   }
 
@@ -460,6 +480,38 @@ public void testFlush() throws Exception {
     Assert.assertTrue(flushService.lastFlushTime > 0);
   }
 
+  @Test
+  public void testBlobCreation() throws Exception {
+    TestContext<?> testContext = testContextFactory.create();
+    SnowflakeStreamingIngestChannelInternal<?> channel1 = addChannel1(testContext);
+    SnowflakeStreamingIngestChannelInternal<?> channel2 = addChannel2(testContext);
+    SnowflakeStreamingIngestChannelInternal<?> channel4 = addChannel4(testContext);
+
+    List<ColumnMetadata> schema = Arrays.asList(createTestIntegerColumn(), createTestTextColumn());
+    channel1.getRowBuffer().setupSchema(schema);
+    channel2.getRowBuffer().setupSchema(schema);
+    channel4.getRowBuffer().setupSchema(schema);
+
+    List<Map<String, Object>> rows1 =
+        RowSetBuilder.newBuilder()
+            .addColumn("COLINT", 11)
+            .addColumn("COLCHAR", "bob")
+            .newRow()
+            .addColumn("COLINT", 22)
+            .addColumn("COLCHAR", "bob")
+            .build();
+
+    channel1.insertRows(rows1, "offset1");
+    channel2.insertRows(rows1, "offset2");
+    channel4.insertRows(rows1, "offset4");
+
+    FlushService<?> flushService = testContext.flushService;
+
+    // Force = true flushes
+    flushService.flush(true).get();
+    Mockito.verify(flushService, Mockito.atLeast(2)).buildAndUpload(Mockito.any(), Mockito.any());
+  }
+
   @Test
   public void testBuildAndUpload() throws Exception {
     TestContext<?> testContext = testContextFactory.create();

From 54cf2d19da866aa5e3f9d7c22357ac4ceb137219 Mon Sep 17 00:00:00 2001
From: Snyk bot <snyk-bot@snyk.io>
Date: Thu, 30 Mar 2023 01:26:07 +0800
Subject: [PATCH 152/356] [Snyk] Security upgrade
 org.apache.hadoop:hadoop-common from 3.3.4 to 3.3.5 (#370)

* fix: pom.xml to reduce vulnerabilities

The following vulnerabilities are fixed with an upgrade:
- https://snyk.io/vuln/SNYK-JAVA-ORGECLIPSEJETTY-2945452

* fix error

---------

Co-authored-by: Toby Zhang <toby.zhang@snowflake.com>
---
 pom.xml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 2c1910e55..8574481ed 100644
--- a/pom.xml
+++ b/pom.xml
@@ -43,9 +43,10 @@
     <commonsio.version>2.8.0</commonsio.version>
     <commonslang3.version>3.12.0</commonslang3.version>
     <commonslogging.version>1.2</commonslogging.version>
+    <commonstext.version>1.10.0</commonstext.version>
     <fasterxml.version>2.14.0</fasterxml.version>
     <guava.version>31.1-jre</guava.version>
-    <hadoop.version>3.3.4</hadoop.version>
+    <hadoop.version>3.3.5</hadoop.version>
     <jacoco.skip.instrument>true</jacoco.skip.instrument>
     <jacoco.version>0.8.5</jacoco.version>
     <maven.compiler.source>1.8</maven.compiler.source>
@@ -126,6 +127,11 @@
         <artifactId>commons-lang3</artifactId>
         <version>${commonslang3.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.commons</groupId>
+        <artifactId>commons-text</artifactId>
+        <version>${commonstext.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-common</artifactId>

From 1a9444af0ded898c5ad53986c57fc799ad279ece Mon Sep 17 00:00:00 2001
From: Tyler Jones <tyler.jones@snowflake.com>
Date: Thu, 30 Mar 2023 00:18:05 +0000
Subject: [PATCH 153/356] Resolves boundcastle related linkage errors

---
 linkage-checker-exclusion-rules.xml | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/linkage-checker-exclusion-rules.xml b/linkage-checker-exclusion-rules.xml
index ecb2afe9d..892f06298 100644
--- a/linkage-checker-exclusion-rules.xml
+++ b/linkage-checker-exclusion-rules.xml
@@ -7,6 +7,14 @@
     <Target><Package name="org.conscrypt"/></Target>
     <Reason>Optional</Reason>
   </LinkageError>
+  <LinkageError>
+    <Target><Package name="com.google.crypto.tink.subtle.X25519"/></Target>
+    <Reason>Optional</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Package name="com.google.crypto.tink.subtle"/></Target>
+    <Reason>Optional</Reason>
+  </LinkageError>
   <LinkageError>
     <Target><Package name="com.sun.jna"/></Target>
     <Reason>Optional</Reason>
@@ -23,6 +31,25 @@
     <Source><Package name="net.snowflake.client.jdbc.internal"/></Source>
     <Reason>Unknown: Likely the referenced class is not used in the jdbc package</Reason>
   </LinkageError>
-
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.com.nimbusds.jose.crypto.impl.LegacyAESGCM"/></Source>
+    <Reason>Not Used, appears to be Legacy</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.com.nimbusds.jose.jwk"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.com.nimbusds.jose.crypto.bc.BouncyCastleProviderSingleton"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.com.nimbusds.jose.util.X509CertChainUtils"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.com.nimbusds.jose.crypto.bc.BouncyCastleFIPSProviderSingleton"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
 </LinkageCheckerFilter>
 

From ff3b375ef239f764900508b2d4dd797cb4559ee3 Mon Sep 17 00:00:00 2001
From: Tyler Jones <tyler.jones@snowflake.com>
Date: Thu, 30 Mar 2023 19:50:44 +0000
Subject: [PATCH 154/356] Iteration 2: Addresses sfc-gh-pbennes feedback

---
 linkage-checker-exclusion-rules.xml | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/linkage-checker-exclusion-rules.xml b/linkage-checker-exclusion-rules.xml
index 892f06298..0139a7d76 100644
--- a/linkage-checker-exclusion-rules.xml
+++ b/linkage-checker-exclusion-rules.xml
@@ -7,10 +7,6 @@
     <Target><Package name="org.conscrypt"/></Target>
     <Reason>Optional</Reason>
   </LinkageError>
-  <LinkageError>
-    <Target><Package name="com.google.crypto.tink.subtle.X25519"/></Target>
-    <Reason>Optional</Reason>
-  </LinkageError>
   <LinkageError>
     <Target><Package name="com.google.crypto.tink.subtle"/></Target>
     <Reason>Optional</Reason>
@@ -32,7 +28,7 @@
     <Reason>Unknown: Likely the referenced class is not used in the jdbc package</Reason>
   </LinkageError>
   <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.com.nimbusds.jose.crypto.impl.LegacyAESGCM"/></Source>
+    <Source><Class name="net.snowflake.ingest.internal.com.nimbusds.jose.crypto.impl.LegacyAESGCM"/></Source>
     <Reason>Not Used, appears to be Legacy</Reason>
   </LinkageError>
   <LinkageError>
@@ -40,15 +36,11 @@
     <Reason>Not Used</Reason>
   </LinkageError>
   <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.com.nimbusds.jose.crypto.bc.BouncyCastleProviderSingleton"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.com.nimbusds.jose.util.X509CertChainUtils"/></Source>
+    <Source><Package name="net.snowflake.ingest.internal.com.nimbusds.jose.crypto.bc"/></Source>
     <Reason>Not Used</Reason>
   </LinkageError>
   <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.com.nimbusds.jose.crypto.bc.BouncyCastleFIPSProviderSingleton"/></Source>
+    <Source><Class name="net.snowflake.ingest.internal.com.nimbusds.jose.util.X509CertChainUtils"/></Source>
     <Reason>Not Used</Reason>
   </LinkageError>
 </LinkageCheckerFilter>

From 310fbf146c4095b88bd3dcab63555591245141ee Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Fri, 31 Mar 2023 17:05:11 -0700
Subject: [PATCH 155/356] init impl

---
 .../streaming/internal/BlobMetadata.java      | 31 +++++-
 .../streaming/internal/FlushService.java      | 78 ++++++++-------
 .../streaming/internal/RegisterService.java   | 18 ++--
 ...nowflakeStreamingIngestClientInternal.java | 99 ++++++++++---------
 .../streaming/internal/FlushServiceTest.java  | 59 +++++------
 .../internal/RegisterServiceTest.java         | 19 ++--
 .../SnowflakeStreamingIngestClientTest.java   | 71 ++++++-------
 7 files changed, 207 insertions(+), 168 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index 8b0dd52dd..39b1dbca1 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -6,27 +6,38 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import java.util.List;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ParameterProvider;
 
+import java.util.List;
+
 /** Metadata for a blob that sends to Snowflake as part of the register blob request */
 class BlobMetadata {
+  public static final long DEFAULT_BLOB_LATENCY = -1;
+
   private final String path;
   private final String md5;
   private final Constants.BdecVersion bdecVersion;
   private final List<ChunkMetadata> chunks;
 
+  // latencies
+  private final long buildLatencyMs;
+  private final long uploadLatencyMs;
+
+  // used for testing only
   BlobMetadata(String path, String md5, List<ChunkMetadata> chunks) {
-    this(path, md5, ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT, chunks);
+    this(path, md5, ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT, chunks, DEFAULT_BLOB_LATENCY, DEFAULT_BLOB_LATENCY);
   }
 
   BlobMetadata(
-      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks) {
+      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks, long buildLatencyMs, long uploadLatencyMs) {
     this.path = path;
     this.md5 = md5;
     this.bdecVersion = bdecVersion;
     this.chunks = chunks;
+
+    this.buildLatencyMs = buildLatencyMs;
+    this.uploadLatencyMs = uploadLatencyMs;
   }
 
   @JsonIgnore
@@ -54,9 +65,19 @@ byte getVersionByte() {
     return bdecVersion.toByte();
   }
 
+  @JsonProperty("build_latency_ms")
+  long getBuildLatencyMs() {
+    return this.buildLatencyMs;
+  }
+
+  @JsonProperty("upload_latency_ms")
+  long getUploadLatencyMs() {
+    return this.uploadLatencyMs;
+  }
+
   /** Create {@link BlobMetadata}. */
   static BlobMetadata createBlobMetadata(
-      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks) {
-    return new BlobMetadata(path, md5, bdecVersion, chunks);
+      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks, long buildLatencyMs, long uploadLatencyMs) {
+    return new BlobMetadata(path, md5, bdecVersion, chunks, buildLatencyMs, uploadLatencyMs);
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 7758e30e6..82d6a25f8 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -4,14 +4,21 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
-import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
-import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
-
 import com.codahale.metrics.Timer;
+import net.snowflake.client.jdbc.SnowflakeSQLException;
+import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.util.VisibleForTesting;
+import org.apache.arrow.vector.VectorSchemaRoot;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.security.InvalidAlgorithmParameterException;
@@ -35,19 +42,13 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import net.snowflake.client.jdbc.SnowflakeSQLException;
-import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.util.VisibleForTesting;
-import org.apache.arrow.vector.VectorSchemaRoot;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
+import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
+import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
 /**
  * Responsible for flushing data from client to Snowflake tables. When a flush is triggered, it will
@@ -238,12 +239,13 @@ private void logFlushTask(boolean isForce, long timeDiffMillis) {
 
   /**
    * Registers blobs with Snowflake
+   * @param flushJobStartTime When the flush job began
    *
    * @return
    */
-  private CompletableFuture<Void> registerFuture() {
+  private CompletableFuture<Void> registerFuture(long flushJobStartTime) {
     return CompletableFuture.runAsync(
-        () -> this.registerService.registerBlobs(latencyTimerContextMap), this.registerWorker);
+        () -> this.registerService.registerBlobs(latencyTimerContextMap, flushJobStartTime), this.registerWorker);
   }
 
   /**
@@ -258,6 +260,8 @@ private CompletableFuture<Void> registerFuture() {
    */
   CompletableFuture<Void> flush(boolean isForce) {
     long timeDiffMillis = System.currentTimeMillis() - this.lastFlushTime;
+    long flushJobStartTime = System.currentTimeMillis();
+
     if (isForce
         || (!DISABLE_BACKGROUND_FLUSH
             && !this.isTestMode
@@ -267,7 +271,7 @@ CompletableFuture<Void> flush(boolean isForce) {
 
       return this.statsFuture()
           .thenCompose((v) -> this.distributeFlush(isForce, timeDiffMillis))
-          .thenCompose((v) -> this.registerFuture());
+          .thenCompose((v) -> this.registerFuture(flushJobStartTime));
     }
     return this.statsFuture();
   }
@@ -487,10 +491,10 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
 
     // Construct the blob along with the metadata of the blob
     BlobBuilder.Blob blob = BlobBuilder.constructBlobAndMetadata(filePath, blobData, bdecVersion);
-    if (buildContext != null) {
-      buildContext.stop();
-    }
-    return upload(filePath, blob.blobBytes, blob.chunksMetadataList);
+
+    return buildContext != null ?
+      upload(filePath, blob.blobBytes, blob.chunksMetadataList, buildContext.stop()) :
+      upload(filePath, blob.blobBytes, blob.chunksMetadataList, BlobMetadata.DEFAULT_BLOB_LATENCY);
   }
 
   /**
@@ -499,33 +503,35 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
    * @param filePath full path of the blob file
    * @param blob blob data
    * @param metadata a list of chunk metadata
+   * @param buildLatencyMs the build latency for the blob
    * @return BlobMetadata object used to create the register blob request
    */
-  BlobMetadata upload(String filePath, byte[] blob, List<ChunkMetadata> metadata)
+  BlobMetadata upload(String filePath, byte[] blob, List<ChunkMetadata> metadata, long buildLatencyMs)
       throws NoSuchAlgorithmException {
     logger.logInfo("Start uploading file={}, size={}", filePath, blob.length);
     long startTime = System.currentTimeMillis();
 
     Timer.Context uploadContext = Utils.createTimerContext(this.owningClient.uploadLatency);
+    long uploadLatencyMs = BlobMetadata.DEFAULT_BLOB_LATENCY;
 
     this.targetStage.put(filePath, blob);
 
+    logger.logInfo(
+            "Finish uploading file={}, size={}, timeInMillis={}",
+            filePath,
+            blob.length,
+            System.currentTimeMillis() - startTime);
+
     if (uploadContext != null) {
-      uploadContext.stop();
+      uploadLatencyMs = uploadContext.stop();
       this.owningClient.uploadThroughput.mark(blob.length);
       this.owningClient.blobSizeHistogram.update(blob.length);
       this.owningClient.blobRowCountHistogram.update(
           metadata.stream().mapToLong(i -> i.getEpInfo().getRowCount()).sum());
     }
 
-    logger.logInfo(
-        "Finish uploading file={}, size={}, timeInMillis={}",
-        filePath,
-        blob.length,
-        System.currentTimeMillis() - startTime);
-
     return BlobMetadata.createBlobMetadata(
-        filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata);
+        filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata, buildLatencyMs, uploadLatencyMs);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index 44b90987b..93db9a7a0 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -4,10 +4,11 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
-
 import com.codahale.metrics.Timer;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.Utils;
+
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -17,9 +18,9 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.Utils;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
 /**
  * Register one or more blobs to the targeted Snowflake table, it will be done using the dedicated
@@ -79,9 +80,10 @@ void addBlobs(List<Pair<FlushService.BlobData<T>, CompletableFuture<BlobMetadata
    * the ordering is maintained across independent blobs in the same channel.
    *
    * @param latencyTimerContextMap the map that stores the latency timer for each blob
+   * @param flushJobStartTime When the flush job began
    * @return a list of blob names that have errors during registration
    */
-  List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyTimerContextMap) {
+  List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyTimerContextMap, long flushJobStartTime) {
     List<FlushService.BlobData<T>> errorBlobs = new ArrayList<>();
     if (!this.blobsList.isEmpty()) {
       // Will skip and try again later if someone else is holding the lock
@@ -197,7 +199,7 @@ List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyT
                 Utils.createTimerContext(this.owningClient.registerLatency);
 
             // Register the blobs, and invalidate any channels that return a failure status code
-            this.owningClient.registerBlobs(blobs);
+            this.owningClient.registerBlobs(blobs, flushJobStartTime);
 
             if (registerContext != null) {
               registerContext.stop();
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 10d4d2377..66b2a2bdd 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -4,28 +4,6 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CHANNEL_STATUS;
-import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_OPEN_CHANNEL;
-import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_REGISTER_BLOB;
-import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
-import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.sleepForRetry;
-import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.COMMIT_MAX_RETRY_COUNT;
-import static net.snowflake.ingest.utils.Constants.COMMIT_RETRY_INTERVAL_IN_MS;
-import static net.snowflake.ingest.utils.Constants.ENABLE_TELEMETRY_TO_SF;
-import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
-import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
-import static net.snowflake.ingest.utils.Constants.OPEN_CHANNEL_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
-import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JMX_METRIC_PREFIX;
-import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JVM_MEMORY_AND_THREAD_METRICS_REGISTRY;
-import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_SHARED_METRICS_REGISTRY;
-import static net.snowflake.ingest.utils.Constants.STREAMING_INGEST_TELEMETRY_UPLOAD_INTERVAL_IN_SEC;
-import static net.snowflake.ingest.utils.Constants.USER;
-
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.MetricFilter;
@@ -37,6 +15,26 @@
 import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
 import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
+import net.snowflake.ingest.connection.IngestResponseException;
+import net.snowflake.ingest.connection.RequestBuilder;
+import net.snowflake.ingest.connection.TelemetryService;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.HttpUtil;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.ParameterProvider;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.SnowflakeURL;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
 import java.io.IOException;
 import java.security.KeyPair;
 import java.security.NoSuchAlgorithmException;
@@ -56,25 +54,28 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
-import javax.management.MalformedObjectNameException;
-import javax.management.ObjectName;
-import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
-import net.snowflake.ingest.connection.IngestResponseException;
-import net.snowflake.ingest.connection.RequestBuilder;
-import net.snowflake.ingest.connection.TelemetryService;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.HttpUtil;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.ParameterProvider;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.SnowflakeURL;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
+
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CHANNEL_STATUS;
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_OPEN_CHANNEL;
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_REGISTER_BLOB;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.sleepForRetry;
+import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.COMMIT_MAX_RETRY_COUNT;
+import static net.snowflake.ingest.utils.Constants.COMMIT_RETRY_INTERVAL_IN_MS;
+import static net.snowflake.ingest.utils.Constants.ENABLE_TELEMETRY_TO_SF;
+import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
+import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
+import static net.snowflake.ingest.utils.Constants.OPEN_CHANNEL_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
+import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JMX_METRIC_PREFIX;
+import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JVM_MEMORY_AND_THREAD_METRICS_REGISTRY;
+import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_SHARED_METRICS_REGISTRY;
+import static net.snowflake.ingest.utils.Constants.STREAMING_INGEST_TELEMETRY_UPLOAD_INTERVAL_IN_SEC;
+import static net.snowflake.ingest.utils.Constants.USER;
 
 /**
  * The first version of implementation for SnowflakeStreamingIngestClient. The client internally
@@ -404,9 +405,10 @@ ChannelsStatusResponse getChannelsStatus(
    * Register the uploaded blobs to a Snowflake table
    *
    * @param blobs list of uploaded blobs
+   * @param flushJobStartTime When the flush job began
    */
-  void registerBlobs(List<BlobMetadata> blobs) {
-    this.registerBlobs(blobs, 0);
+  void registerBlobs(List<BlobMetadata> blobs, long flushJobStartTime) {
+    this.registerBlobs(blobs, 0, flushJobStartTime);
   }
 
   /**
@@ -414,8 +416,9 @@ void registerBlobs(List<BlobMetadata> blobs) {
    *
    * @param blobs list of uploaded blobs
    * @param executionCount Number of times this call has been attempted, used to track retries
+   * @param flushJobStartTime When the flush job began
    */
-  void registerBlobs(List<BlobMetadata> blobs, final int executionCount) {
+  void registerBlobs(List<BlobMetadata> blobs, final int executionCount, long flushJobStartTime) {
     logger.logInfo(
         "Register blob request preparing for blob={}, client={}, executionCount={}",
         blobs.stream().map(BlobMetadata::getPath).collect(Collectors.toList()),
@@ -429,6 +432,8 @@ void registerBlobs(List<BlobMetadata> blobs, final int executionCount) {
           "request_id", this.flushService.getClientPrefix() + "_" + counter.getAndIncrement());
       payload.put("blobs", blobs);
       payload.put("role", this.role);
+      payload.put("flush_start_timestamp", flushJobStartTime);
+      payload.put("register_start_timestamp", System.currentTimeMillis());
 
       response =
           executeWithRetries(
@@ -520,7 +525,7 @@ void registerBlobs(List<BlobMetadata> blobs, final int executionCount) {
         throw new SFException(ErrorCode.INTERNAL_ERROR, "Failed to retry queue full chunks");
       }
       sleepForRetry(executionCount);
-      this.registerBlobs(retryBlobs, executionCount + 1);
+      this.registerBlobs(retryBlobs, executionCount + 1, flushJobStartTime);
     }
   }
 
@@ -572,7 +577,9 @@ List<BlobMetadata> getRetryBlobs(
                     blobMetadata.getPath(),
                     blobMetadata.getMD5(),
                     blobMetadata.getVersion(),
-                    relevantChunks));
+                    relevantChunks,
+                    blobMetadata.getBuildLatencyMs(),
+                    blobMetadata.getUploadLatencyMs()));
           }
         });
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 0c485b94b..7f78b723a 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -1,14 +1,27 @@
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
-import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
-
 import com.fasterxml.jackson.databind.ObjectMapper;
+import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.Cryptor;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.ParameterProvider;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.math.BigInteger;
@@ -32,26 +45,14 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TimeZone;
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.Cryptor;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.ParameterProvider;
-import net.snowflake.ingest.utils.SFException;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Mockito;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
+import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
 
 @RunWith(Parameterized.class)
 public class FlushServiceTest {
@@ -599,7 +600,7 @@ public void testBuildAndUpload() throws Exception {
     final ArgumentCaptor<List<ChunkMetadata>> metadataCaptor = ArgumentCaptor.forClass(List.class);
 
     Mockito.verify(testContext.flushService)
-        .upload(nameCaptor.capture(), blobCaptor.capture(), metadataCaptor.capture());
+        .upload(nameCaptor.capture(), blobCaptor.capture(), metadataCaptor.capture(), System.currentTimeMillis());
     Assert.assertEquals("file_name", nameCaptor.getValue());
 
     ChunkMetadata metadataResult = metadataCaptor.getValue().get(0);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
index d2d2d3301..418d183be 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
@@ -1,6 +1,9 @@
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
+import net.snowflake.ingest.utils.Pair;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -9,10 +12,8 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import net.snowflake.ingest.utils.Pair;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
 
 public class RegisterServiceTest {
 
@@ -26,7 +27,7 @@ public void testRegisterService() {
             CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null)));
     rs.addBlobs(Collections.singletonList(blobFuture));
     Assert.assertEquals(1, rs.getBlobsList().size());
-    List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null);
+    List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null, System.currentTimeMillis());
     Assert.assertEquals(0, rs.getBlobsList().size());
     Assert.assertEquals(0, errorBlobs.size());
   }
@@ -57,7 +58,7 @@ public void testRegisterServiceTimeoutException() throws Exception {
     rs.addBlobs(Arrays.asList(blobFuture1, blobFuture2));
     Assert.assertEquals(2, rs.getBlobsList().size());
     try {
-      List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null);
+      List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null, System.currentTimeMillis());
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
       Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
@@ -93,7 +94,7 @@ public void testRegisterServiceTimeoutException_testRetries() throws Exception {
     rs.addBlobs(Arrays.asList(blobFuture1, blobFuture2));
     Assert.assertEquals(2, rs.getBlobsList().size());
     try {
-      List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null);
+      List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null, System.currentTimeMillis());
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
       Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
@@ -115,7 +116,7 @@ public void testRegisterServiceNonTimeoutException() {
     rs.addBlobs(Collections.singletonList(blobFuture));
     Assert.assertEquals(1, rs.getBlobsList().size());
     try {
-      List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null);
+      List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null, System.currentTimeMillis());
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
       Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index 93b15398b..b8fcda22e 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -1,37 +1,8 @@
 package net.snowflake.ingest.streaming.internal;
 
-import static java.time.ZoneOffset.UTC;
-import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
-import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
-import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
-import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
-import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
-import static net.snowflake.ingest.utils.Constants.ROLE;
-import static net.snowflake.ingest.utils.Constants.USER;
-import static net.snowflake.ingest.utils.ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS;
-
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.PropertyAccessor;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import java.io.IOException;
-import java.io.StringWriter;
-import java.security.KeyPair;
-import java.security.PrivateKey;
-import java.security.Security;
-import java.time.ZoneOffset;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.stream.Collectors;
 import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils;
 import net.snowflake.client.jdbc.internal.apache.http.HttpEntity;
 import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders;
@@ -66,6 +37,36 @@
 import org.junit.Test;
 import org.mockito.Mockito;
 
+import java.io.IOException;
+import java.io.StringWriter;
+import java.security.KeyPair;
+import java.security.PrivateKey;
+import java.security.Security;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+
+import static java.time.ZoneOffset.UTC;
+import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
+import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
+import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
+import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
+import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
+import static net.snowflake.ingest.utils.Constants.ROLE;
+import static net.snowflake.ingest.utils.Constants.USER;
+import static net.snowflake.ingest.utils.ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS;
+
 public class SnowflakeStreamingIngestClientTest {
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
@@ -674,7 +675,7 @@ public void testRegisterBlobErrorResponse() throws Exception {
     try {
       List<BlobMetadata> blobs =
           Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
-      client.registerBlobs(blobs);
+      client.registerBlobs(blobs, System.currentTimeMillis());
       Assert.fail("Register blob should fail on 404 error");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.REGISTER_BLOB_FAILURE.getMessageCode(), e.getVendorCode());
@@ -721,7 +722,7 @@ public void testRegisterBlobSnowflakeInternalErrorResponse() throws Exception {
     try {
       List<BlobMetadata> blobs =
           Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
-      client.registerBlobs(blobs);
+      client.registerBlobs(blobs, System.currentTimeMillis());
       Assert.fail("Register blob should fail on SF internal error");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.REGISTER_BLOB_FAILURE.getMessageCode(), e.getVendorCode());
@@ -776,7 +777,7 @@ public void testRegisterBlobSuccessResponse() throws Exception {
 
     List<BlobMetadata> blobs =
         Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
-    client.registerBlobs(blobs);
+    client.registerBlobs(blobs, System.currentTimeMillis());
   }
 
   @Test
@@ -862,7 +863,7 @@ public void testRegisterBlobsRetries() throws Exception {
     client.getChannelCache().addChannel(channel2);
     client.getChannelCache().addChannel(channel3);
     client.getChannelCache().addChannel(channel4);
-    client.registerBlobs(blobs);
+    client.registerBlobs(blobs, System.currentTimeMillis());
     Mockito.verify(requestBuilder, Mockito.times(MAX_STREAMING_INGEST_API_CHANNEL_RETRY + 1))
         .generateStreamingIngestPostRequest(Mockito.anyString(), Mockito.any(), Mockito.any());
     Assert.assertFalse(channel1.isValid());
@@ -978,7 +979,7 @@ public void testRegisterBlobsRetriesSucceeds() throws Exception {
     client.getChannelCache().addChannel(channel3);
     client.getChannelCache().addChannel(channel4);
 
-    client.registerBlobs(blobs);
+    client.registerBlobs(blobs, System.currentTimeMillis());
     Mockito.verify(requestBuilder, Mockito.times(2))
         .generateStreamingIngestPostRequest(Mockito.anyString(), Mockito.any(), Mockito.any());
     Assert.assertTrue(channel1.isValid());
@@ -1083,7 +1084,7 @@ public void testRegisterBlobResponseWithInvalidChannel() throws Exception {
 
     List<BlobMetadata> blobs =
         Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
-    client.registerBlobs(blobs);
+    client.registerBlobs(blobs, System.currentTimeMillis());
 
     // Channel2 should be invalidated now
     Assert.assertTrue(channel1.isValid());

From 309deb1276beb50607fe5f70f66758f816296936 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Fri, 31 Mar 2023 17:05:16 -0700
Subject: [PATCH 156/356] autoformatting

---
 .../streaming/internal/BlobMetadata.java      | 25 ++++--
 .../streaming/internal/FlushService.java      | 71 +++++++++-------
 .../streaming/internal/RegisterService.java   | 16 ++--
 ...nowflakeStreamingIngestClientInternal.java | 83 +++++++++----------
 .../streaming/internal/FlushServiceTest.java  | 63 +++++++-------
 .../internal/RegisterServiceTest.java         | 23 ++---
 .../SnowflakeStreamingIngestClientTest.java   | 59 +++++++------
 7 files changed, 183 insertions(+), 157 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index 39b1dbca1..688c21e69 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -6,11 +6,10 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ParameterProvider;
 
-import java.util.List;
-
 /** Metadata for a blob that sends to Snowflake as part of the register blob request */
 class BlobMetadata {
   public static final long DEFAULT_BLOB_LATENCY = -1;
@@ -26,11 +25,22 @@ class BlobMetadata {
 
   // used for testing only
   BlobMetadata(String path, String md5, List<ChunkMetadata> chunks) {
-    this(path, md5, ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT, chunks, DEFAULT_BLOB_LATENCY, DEFAULT_BLOB_LATENCY);
+    this(
+        path,
+        md5,
+        ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT,
+        chunks,
+        DEFAULT_BLOB_LATENCY,
+        DEFAULT_BLOB_LATENCY);
   }
 
   BlobMetadata(
-      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks, long buildLatencyMs, long uploadLatencyMs) {
+      String path,
+      String md5,
+      Constants.BdecVersion bdecVersion,
+      List<ChunkMetadata> chunks,
+      long buildLatencyMs,
+      long uploadLatencyMs) {
     this.path = path;
     this.md5 = md5;
     this.bdecVersion = bdecVersion;
@@ -77,7 +87,12 @@ long getUploadLatencyMs() {
 
   /** Create {@link BlobMetadata}. */
   static BlobMetadata createBlobMetadata(
-      String path, String md5, Constants.BdecVersion bdecVersion, List<ChunkMetadata> chunks, long buildLatencyMs, long uploadLatencyMs) {
+      String path,
+      String md5,
+      Constants.BdecVersion bdecVersion,
+      List<ChunkMetadata> chunks,
+      long buildLatencyMs,
+      long uploadLatencyMs) {
     return new BlobMetadata(path, md5, bdecVersion, chunks, buildLatencyMs, uploadLatencyMs);
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 82d6a25f8..28977208a 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -4,21 +4,14 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import com.codahale.metrics.Timer;
-import net.snowflake.client.jdbc.SnowflakeSQLException;
-import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.util.VisibleForTesting;
-import org.apache.arrow.vector.VectorSchemaRoot;
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
+import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
+import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
+import com.codahale.metrics.Timer;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.security.InvalidAlgorithmParameterException;
@@ -42,13 +35,19 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
-import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
-import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import net.snowflake.client.jdbc.SnowflakeSQLException;
+import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.util.VisibleForTesting;
+import org.apache.arrow.vector.VectorSchemaRoot;
 
 /**
  * Responsible for flushing data from client to Snowflake tables. When a flush is triggered, it will
@@ -239,13 +238,14 @@ private void logFlushTask(boolean isForce, long timeDiffMillis) {
 
   /**
    * Registers blobs with Snowflake
-   * @param flushJobStartTime When the flush job began
    *
+   * @param flushJobStartTime When the flush job began
    * @return
    */
   private CompletableFuture<Void> registerFuture(long flushJobStartTime) {
     return CompletableFuture.runAsync(
-        () -> this.registerService.registerBlobs(latencyTimerContextMap, flushJobStartTime), this.registerWorker);
+        () -> this.registerService.registerBlobs(latencyTimerContextMap, flushJobStartTime),
+        this.registerWorker);
   }
 
   /**
@@ -492,9 +492,10 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
     // Construct the blob along with the metadata of the blob
     BlobBuilder.Blob blob = BlobBuilder.constructBlobAndMetadata(filePath, blobData, bdecVersion);
 
-    return buildContext != null ?
-      upload(filePath, blob.blobBytes, blob.chunksMetadataList, buildContext.stop()) :
-      upload(filePath, blob.blobBytes, blob.chunksMetadataList, BlobMetadata.DEFAULT_BLOB_LATENCY);
+    return buildContext != null
+        ? upload(filePath, blob.blobBytes, blob.chunksMetadataList, buildContext.stop())
+        : upload(
+            filePath, blob.blobBytes, blob.chunksMetadataList, BlobMetadata.DEFAULT_BLOB_LATENCY);
   }
 
   /**
@@ -506,7 +507,8 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
    * @param buildLatencyMs the build latency for the blob
    * @return BlobMetadata object used to create the register blob request
    */
-  BlobMetadata upload(String filePath, byte[] blob, List<ChunkMetadata> metadata, long buildLatencyMs)
+  BlobMetadata upload(
+      String filePath, byte[] blob, List<ChunkMetadata> metadata, long buildLatencyMs)
       throws NoSuchAlgorithmException {
     logger.logInfo("Start uploading file={}, size={}", filePath, blob.length);
     long startTime = System.currentTimeMillis();
@@ -517,10 +519,10 @@ BlobMetadata upload(String filePath, byte[] blob, List<ChunkMetadata> metadata,
     this.targetStage.put(filePath, blob);
 
     logger.logInfo(
-            "Finish uploading file={}, size={}, timeInMillis={}",
-            filePath,
-            blob.length,
-            System.currentTimeMillis() - startTime);
+        "Finish uploading file={}, size={}, timeInMillis={}",
+        filePath,
+        blob.length,
+        System.currentTimeMillis() - startTime);
 
     if (uploadContext != null) {
       uploadLatencyMs = uploadContext.stop();
@@ -531,7 +533,12 @@ BlobMetadata upload(String filePath, byte[] blob, List<ChunkMetadata> metadata,
     }
 
     return BlobMetadata.createBlobMetadata(
-        filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata, buildLatencyMs, uploadLatencyMs);
+        filePath,
+        BlobBuilder.computeMD5(blob),
+        bdecVersion,
+        metadata,
+        buildLatencyMs,
+        uploadLatencyMs);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index 93db9a7a0..b9eed9b6d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -4,11 +4,10 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import com.codahale.metrics.Timer;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.Utils;
+import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
+import com.codahale.metrics.Timer;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -18,9 +17,9 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.Utils;
 
 /**
  * Register one or more blobs to the targeted Snowflake table, it will be done using the dedicated
@@ -83,7 +82,8 @@ void addBlobs(List<Pair<FlushService.BlobData<T>, CompletableFuture<BlobMetadata
    * @param flushJobStartTime When the flush job began
    * @return a list of blob names that have errors during registration
    */
-  List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyTimerContextMap, long flushJobStartTime) {
+  List<FlushService.BlobData<T>> registerBlobs(
+      Map<String, Timer.Context> latencyTimerContextMap, long flushJobStartTime) {
     List<FlushService.BlobData<T>> errorBlobs = new ArrayList<>();
     if (!this.blobsList.isEmpty()) {
       // Will skip and try again later if someone else is holding the lock
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 66b2a2bdd..47f92392f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -4,6 +4,28 @@
 
 package net.snowflake.ingest.streaming.internal;
 
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CHANNEL_STATUS;
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_OPEN_CHANNEL;
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_REGISTER_BLOB;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.sleepForRetry;
+import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.COMMIT_MAX_RETRY_COUNT;
+import static net.snowflake.ingest.utils.Constants.COMMIT_RETRY_INTERVAL_IN_MS;
+import static net.snowflake.ingest.utils.Constants.ENABLE_TELEMETRY_TO_SF;
+import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
+import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
+import static net.snowflake.ingest.utils.Constants.OPEN_CHANNEL_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
+import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JMX_METRIC_PREFIX;
+import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JVM_MEMORY_AND_THREAD_METRICS_REGISTRY;
+import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_SHARED_METRICS_REGISTRY;
+import static net.snowflake.ingest.utils.Constants.STREAMING_INGEST_TELEMETRY_UPLOAD_INTERVAL_IN_SEC;
+import static net.snowflake.ingest.utils.Constants.USER;
+
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.MetricFilter;
@@ -15,26 +37,6 @@
 import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
 import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
-import net.snowflake.ingest.connection.IngestResponseException;
-import net.snowflake.ingest.connection.RequestBuilder;
-import net.snowflake.ingest.connection.TelemetryService;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.HttpUtil;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.ParameterProvider;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.SnowflakeURL;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
-
-import javax.management.MalformedObjectNameException;
-import javax.management.ObjectName;
 import java.io.IOException;
 import java.security.KeyPair;
 import java.security.NoSuchAlgorithmException;
@@ -54,28 +56,25 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
-
-import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CHANNEL_STATUS;
-import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_OPEN_CHANNEL;
-import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_REGISTER_BLOB;
-import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
-import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.sleepForRetry;
-import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.COMMIT_MAX_RETRY_COUNT;
-import static net.snowflake.ingest.utils.Constants.COMMIT_RETRY_INTERVAL_IN_MS;
-import static net.snowflake.ingest.utils.Constants.ENABLE_TELEMETRY_TO_SF;
-import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
-import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
-import static net.snowflake.ingest.utils.Constants.OPEN_CHANNEL_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
-import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JMX_METRIC_PREFIX;
-import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JVM_MEMORY_AND_THREAD_METRICS_REGISTRY;
-import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_SHARED_METRICS_REGISTRY;
-import static net.snowflake.ingest.utils.Constants.STREAMING_INGEST_TELEMETRY_UPLOAD_INTERVAL_IN_SEC;
-import static net.snowflake.ingest.utils.Constants.USER;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
+import net.snowflake.ingest.connection.IngestResponseException;
+import net.snowflake.ingest.connection.RequestBuilder;
+import net.snowflake.ingest.connection.TelemetryService;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.HttpUtil;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.ParameterProvider;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.SnowflakeURL;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
 
 /**
  * The first version of implementation for SnowflakeStreamingIngestClient. The client internally
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 7f78b723a..2219eba7a 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -1,27 +1,14 @@
 package net.snowflake.ingest.streaming.internal;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.Cryptor;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.ParameterProvider;
-import net.snowflake.ingest.utils.SFException;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Mockito;
+import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
+import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
 
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.math.BigInteger;
@@ -45,14 +32,26 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TimeZone;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
-import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.Cryptor;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.ParameterProvider;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
 
 @RunWith(Parameterized.class)
 public class FlushServiceTest {
@@ -600,7 +599,11 @@ public void testBuildAndUpload() throws Exception {
     final ArgumentCaptor<List<ChunkMetadata>> metadataCaptor = ArgumentCaptor.forClass(List.class);
 
     Mockito.verify(testContext.flushService)
-        .upload(nameCaptor.capture(), blobCaptor.capture(), metadataCaptor.capture(), System.currentTimeMillis());
+        .upload(
+            nameCaptor.capture(),
+            blobCaptor.capture(),
+            metadataCaptor.capture(),
+            System.currentTimeMillis());
     Assert.assertEquals("file_name", nameCaptor.getValue());
 
     ChunkMetadata metadataResult = metadataCaptor.getValue().get(0);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
index 418d183be..16cac7345 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
@@ -1,9 +1,6 @@
 package net.snowflake.ingest.streaming.internal;
 
-import net.snowflake.ingest.utils.Pair;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
+import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -12,8 +9,10 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
+import net.snowflake.ingest.utils.Pair;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
 
 public class RegisterServiceTest {
 
@@ -27,7 +26,8 @@ public void testRegisterService() {
             CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null)));
     rs.addBlobs(Collections.singletonList(blobFuture));
     Assert.assertEquals(1, rs.getBlobsList().size());
-    List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null, System.currentTimeMillis());
+    List<FlushService.BlobData<StubChunkData>> errorBlobs =
+        rs.registerBlobs(null, System.currentTimeMillis());
     Assert.assertEquals(0, rs.getBlobsList().size());
     Assert.assertEquals(0, errorBlobs.size());
   }
@@ -58,7 +58,8 @@ public void testRegisterServiceTimeoutException() throws Exception {
     rs.addBlobs(Arrays.asList(blobFuture1, blobFuture2));
     Assert.assertEquals(2, rs.getBlobsList().size());
     try {
-      List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null, System.currentTimeMillis());
+      List<FlushService.BlobData<StubChunkData>> errorBlobs =
+          rs.registerBlobs(null, System.currentTimeMillis());
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
       Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
@@ -94,7 +95,8 @@ public void testRegisterServiceTimeoutException_testRetries() throws Exception {
     rs.addBlobs(Arrays.asList(blobFuture1, blobFuture2));
     Assert.assertEquals(2, rs.getBlobsList().size());
     try {
-      List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null, System.currentTimeMillis());
+      List<FlushService.BlobData<StubChunkData>> errorBlobs =
+          rs.registerBlobs(null, System.currentTimeMillis());
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
       Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
@@ -116,7 +118,8 @@ public void testRegisterServiceNonTimeoutException() {
     rs.addBlobs(Collections.singletonList(blobFuture));
     Assert.assertEquals(1, rs.getBlobsList().size());
     try {
-      List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null, System.currentTimeMillis());
+      List<FlushService.BlobData<StubChunkData>> errorBlobs =
+          rs.registerBlobs(null, System.currentTimeMillis());
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
       Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index b8fcda22e..08c30edcc 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -1,8 +1,37 @@
 package net.snowflake.ingest.streaming.internal;
 
+import static java.time.ZoneOffset.UTC;
+import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
+import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
+import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
+import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
+import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
+import static net.snowflake.ingest.utils.Constants.ROLE;
+import static net.snowflake.ingest.utils.Constants.USER;
+import static net.snowflake.ingest.utils.ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS;
+
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.PropertyAccessor;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.security.KeyPair;
+import java.security.PrivateKey;
+import java.security.Security;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
 import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils;
 import net.snowflake.client.jdbc.internal.apache.http.HttpEntity;
 import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders;
@@ -37,36 +66,6 @@
 import org.junit.Test;
 import org.mockito.Mockito;
 
-import java.io.IOException;
-import java.io.StringWriter;
-import java.security.KeyPair;
-import java.security.PrivateKey;
-import java.security.Security;
-import java.time.ZoneOffset;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.stream.Collectors;
-
-import static java.time.ZoneOffset.UTC;
-import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
-import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
-import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
-import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
-import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
-import static net.snowflake.ingest.utils.Constants.ROLE;
-import static net.snowflake.ingest.utils.Constants.USER;
-import static net.snowflake.ingest.utils.ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS;
-
 public class SnowflakeStreamingIngestClientTest {
   private static final ObjectMapper objectMapper = new ObjectMapper();
 

From 40cfd08878dee3bede9353c59f6d9683d12b8518 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Mon, 3 Apr 2023 16:33:03 -0700
Subject: [PATCH 157/356] fix test

---
 .../streaming/internal/BlobMetadata.java      |  3 +-
 .../streaming/internal/FlushServiceTest.java  | 60 ++++++++++---------
 2 files changed, 33 insertions(+), 30 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index 688c21e69..90b94e527 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -6,10 +6,11 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import java.util.List;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ParameterProvider;
 
+import java.util.List;
+
 /** Metadata for a blob that sends to Snowflake as part of the register blob request */
 class BlobMetadata {
   public static final long DEFAULT_BLOB_LATENCY = -1;
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 2219eba7a..fa3b6f71a 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -1,14 +1,28 @@
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
-import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
-
 import com.fasterxml.jackson.databind.ObjectMapper;
+import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.Cryptor;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.ParameterProvider;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.ArgumentCaptor;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.math.BigInteger;
@@ -32,26 +46,14 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TimeZone;
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.Cryptor;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.ParameterProvider;
-import net.snowflake.ingest.utils.SFException;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Mockito;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
+import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
 
 @RunWith(Parameterized.class)
 public class FlushServiceTest {
@@ -603,7 +605,7 @@ public void testBuildAndUpload() throws Exception {
             nameCaptor.capture(),
             blobCaptor.capture(),
             metadataCaptor.capture(),
-            System.currentTimeMillis());
+            ArgumentMatchers.anyLong());
     Assert.assertEquals("file_name", nameCaptor.getValue());
 
     ChunkMetadata metadataResult = metadataCaptor.getValue().get(0);

From aa84f6009fb4b308a973b64e39ee0ca4edf33500 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Mon, 3 Apr 2023 16:33:05 -0700
Subject: [PATCH 158/356] autoformatting

---
 .../streaming/internal/BlobMetadata.java      |  3 +-
 .../streaming/internal/FlushServiceTest.java  | 59 +++++++++----------
 2 files changed, 30 insertions(+), 32 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index 90b94e527..688c21e69 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -6,11 +6,10 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ParameterProvider;
 
-import java.util.List;
-
 /** Metadata for a blob that sends to Snowflake as part of the register blob request */
 class BlobMetadata {
   public static final long DEFAULT_BLOB_LATENCY = -1;
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index fa3b6f71a..0577ac107 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -1,28 +1,14 @@
 package net.snowflake.ingest.streaming.internal;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.Cryptor;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.ParameterProvider;
-import net.snowflake.ingest.utils.SFException;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.mockito.ArgumentCaptor;
-import org.mockito.ArgumentMatchers;
-import org.mockito.Mockito;
+import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
+import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
 
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.math.BigInteger;
@@ -46,14 +32,27 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TimeZone;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
-import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.Cryptor;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.ParameterProvider;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.ArgumentCaptor;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
 
 @RunWith(Parameterized.class)
 public class FlushServiceTest {

From dedf0b5dff177b293ed30e0d035f423265c9714b Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Wed, 5 Apr 2023 12:32:45 -0700
Subject: [PATCH 159/356] swapping to devvm

---
 profile.json.example                          | 22 ++++-----
 .../SnowflakeStreamingIngestExample.java      | 24 ++++-----
 .../streaming/internal/FlushService.java      | 42 ++++++++--------
 .../internal/StreamingIngestStage.java        | 39 ++++++++-------
 .../java/net/snowflake/ingest/TestUtils.java  | 49 ++++++++++---------
 5 files changed, 90 insertions(+), 86 deletions(-)

diff --git a/profile.json.example b/profile.json.example
index 46dbd42f0..14007cfd0 100644
--- a/profile.json.example
+++ b/profile.json.example
@@ -1,15 +1,15 @@
 {
-  "user": "user name",
-  "url": "https://account_name.snowflakecomputing.com:443",
-  "account": "account_name",
-  "private_key": "PEM Private Key",
-  "port": 443,
-  "host": "account_name.snowflakecomputing.com",
-  "schema": "schema",
+  "user": "admin",
+  "url": "http://snowflake.dev.local:8082/",
+  "account": "snowflake",
+  "private_key": "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCsFG3wgtImSV01sl3dbvNotwPTbFiOWz/W1zo/exO840uNyI1VYQ++Dzfxw5Vpp7KwELmPD8xZC0S4FmfYxSJa293qHDYM5sVnH4VYl/Jx5y8rubL9Ye1KQGgW7+uW7O/vgUzlwakMnwVd6Yrr9mPM1HF/E6uTPppi9sTgiylOwZb5CKoctXmZyi3oemotK7r797id+612HSNPxgmrdp6Sj7ckDDABnchTe8JqDTcAlupJApZCUSywZg15LB/lBiGJrGFp0MaxtrvZl1sr+OEqCflKyiRsjvVRnF9YOHu2Msa6FG3/xcUTkNDK4wwiqSgzfhM2qQpwN6nD8IahPpAdAgMBAAECggEABTI8u6WHqQZQHT38naIrEX9tSyYbankQ7XDkXmReDTcj4e0rb2eV7RxSiSdpzJ32xGBM6igE+K9hbNmoWyBg0DOReO9PJoaVcW6/PRShipT7lKtv3zzkyHg3bHHkQFN5T1ctNkbpzurCY7qYqlVvYBEOOFMkvDaIvPJMsaciITGkqtK8nI64BMKH2zR8pZt4ZgIczN5PojXjbhqubWnaIWEle+AOV08BQJoH/63nxZj6uKq8UnER0jPp2Hz/+bHFPv6McYrweo+VFI6OaLTAWk8UKGqoMVkJISaDb/uBiM6vISEJkLKqRE1zeaz+cBWQSy0fCDmgAmGKybq8xpvWdQKBgQDV42p6CESyyriWS7U2iQ+cP/W4NOEfQCS0BZ1ss5rrrDAKK2lxUgNdQiwfmH1V801dNaLosvb+0UtNQQI6kG47BneFvw/1OT/2CE0GqAO7HvY3qIgy/+WaICZEUVV0mE82+irX0Wi0SusBx1qVtkOe/orjU/JdlaRtSVhkYF+GFwKBgQDN9b5Xfv0vZdRTeYpc5Oc2NzhbWhVyEpaOmGfuEEVQbaWL+2lpQyf+MuhUmCOtX3ZEjSuMSr3hYrI5Hg2iwB95wK2fPrCELdD7WpYjdGnzlfO+mDEMM3SlSpphEy7k7PaV2onmrRrWtMNUwPtyiFZ7RRDpmWlrAnm7FeK1Sfnv6wKBgGexJMGcmJGFLg/PDD8wG11ItLtlB267DhAoWLWGLSjsBS4o9al2dSMVHYcDyl+M4Ii7pArkGFJY7QS663Ww53++FtvEUNw1sQh55b3AXm6tB/jbv4vZ+1nJFEQymhc4b87bYUHXx8cAOwZVu1ixT/8YsASdgfZUwa4pzKx8/FJpAoGBAIp77Aok5tSR4ZxnIrOnbhe0NX6nKbhT60viCi+2XQThVOi0mYIfl6qCTFllGsgeYgVh7qAcOWRs2m0xWfXOvNs+xP9IRaP6soPvuvgH9J3Ge+fxqTkM+CeT7A3NukBPXNYR5ZtnVZ58WMKKKAgwyAWGwQJxFsLPHU7APnmrThUVAoGBAM8MUDbaWWITWjQmdx44spukPJ/tt5bPMocarJbb9htPLgqZtpun+uHoBRjbSweJf1jDe/mKTgClR93ZjZG69TnbGQwrKnxZv1Hn1NY7kUnPb+uYs4miGr9tjQ48xAOlcyzfrU1UPXWlWkDFN9GicaNasa11F3npvf8MUBI8d8So",
+  "port": 8082,
+  "host": "snowflake.dev.local",
+  "schema": "PUBLIC",
   "scheme": "https",
-  "database": "database_name",
-  "connect_string": "jdbc:snowflake://account_name.snowflakecomputing.com:443",
+  "database": "REPLICATION",
+  "connect_string": "jdbc:snowflake://snowflake.dev.local:8082",
   "ssl": "on",
-  "warehouse": "warehouse_name",
-  "role": "role_name"
+  "warehouse": "compute_service_warehouse",
+  "role": "accountadmin"
 }
\ No newline at end of file
diff --git a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
index 2b12aba9f..e598709b0 100644
--- a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
+++ b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
@@ -6,17 +6,18 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import net.snowflake.ingest.streaming.InsertValidationResponse;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Properties;
-import net.snowflake.ingest.streaming.InsertValidationResponse;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
 
 /**
  * Example on how to use the Streaming Ingest client APIs.
@@ -27,8 +28,9 @@ public class SnowflakeStreamingIngestExample {
   // Please follow the example in profile_streaming.json.example to see the required properties, or
   // if you have already set up profile.json with Snowpipe before, all you need is to add the "role"
   // property.
-  private static String PROFILE_PATH = "profile.json";
+  private static String PROFILE_PATH = "profile.json.example";
   private static final ObjectMapper mapper = new ObjectMapper();
+  private static final String TABLE_NAME = "revi_ingest_1";
 
   public static void main(String[] args) throws Exception {
     Properties props = new Properties();
@@ -47,10 +49,10 @@ public static void main(String[] args) throws Exception {
       // db/schema/table needs to be present
       // Example: create or replace table MY_TABLE(c1 number);
       OpenChannelRequest request1 =
-          OpenChannelRequest.builder("MY_CHANNEL")
-              .setDBName("MY_DATABASE")
-              .setSchemaName("MY_SCHEMA")
-              .setTableName("MY_TABLE")
+          OpenChannelRequest.builder("revi_channel")
+              .setDBName(props.getProperty("database"))
+              .setSchemaName(props.getProperty("schema"))
+              .setTableName(TABLE_NAME)
               .setOnErrorOption(
                   OpenChannelRequest.OnErrorOption.CONTINUE) // Another ON_ERROR option is ABORT
               .build();
@@ -78,7 +80,7 @@ public static void main(String[] args) throws Exception {
       // If needed, you can check the offset_token registered in Snowflake to make sure everything
       // is committed
       final int expectedOffsetTokenInSnowflake = totalRowsInTable - 1; // 0 based offset_token
-      final int maxRetries = 10;
+      final int maxRetries = 5;
       int retryCount = 0;
 
       do {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 28977208a..84b2cb034 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -4,14 +4,21 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
-import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
-import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
-
 import com.codahale.metrics.Timer;
+import net.snowflake.client.jdbc.SnowflakeSQLException;
+import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.util.VisibleForTesting;
+import org.apache.arrow.vector.VectorSchemaRoot;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.security.InvalidAlgorithmParameterException;
@@ -35,19 +42,13 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import net.snowflake.client.jdbc.SnowflakeSQLException;
-import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.util.VisibleForTesting;
-import org.apache.arrow.vector.VectorSchemaRoot;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
+import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
+import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
 /**
  * Responsible for flushing data from client to Snowflake tables. When a flush is triggered, it will
@@ -348,7 +349,6 @@ void distributeFlushTasks() {
       List<List<ChannelData<T>>> blobData = new ArrayList<>();
       float totalBufferSizeInBytes = 0F;
       final String filePath = getFilePath(this.targetStage.getClientPrefix());
-
       // Distribute work at table level, split the blob if reaching the blob size limit or the
       // channel has different encryption key ids
       while (itr.hasNext() || !leftoverChannelsDataPerTable.isEmpty()) {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index d3b285cd5..ebceda731 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -4,24 +4,6 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.client.core.Constants.CLOUD_STORAGE_CREDENTIALS_EXPIRED;
-import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CLIENT_CONFIGURE;
-import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
-import static net.snowflake.ingest.utils.Constants.CLIENT_CONFIGURE_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
-import static net.snowflake.ingest.utils.HttpUtil.generateProxyPropertiesForJDBC;
-
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Paths;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
 import net.snowflake.client.core.OCSPMode;
 import net.snowflake.client.jdbc.SnowflakeFileTransferAgent;
 import net.snowflake.client.jdbc.SnowflakeFileTransferConfig;
@@ -41,6 +23,25 @@
 import net.snowflake.ingest.utils.Utils;
 import org.apache.arrow.util.VisibleForTesting;
 
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+
+import static net.snowflake.client.core.Constants.CLOUD_STORAGE_CREDENTIALS_EXPIRED;
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CLIENT_CONFIGURE;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
+import static net.snowflake.ingest.utils.Constants.CLIENT_CONFIGURE_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
+import static net.snowflake.ingest.utils.HttpUtil.generateProxyPropertiesForJDBC;
+
 /** Handles uploading files to the Snowflake Streaming Ingest Stage */
 class StreamingIngestStage {
   private static final ObjectMapper mapper = new ObjectMapper();
@@ -396,7 +397,7 @@ void putLocal(String fullFilePath, byte[] data) {
 
     InputStream input = new ByteArrayInputStream(data);
     try {
-      String stageLocation = this.fileTransferMetadataWithAge.localLocation;
+      String stageLocation =  this.fileTransferMetadataWithAge.localLocation; 
       File destFile = Paths.get(stageLocation, fullFilePath).toFile();
       FileUtils.copyInputStreamToFile(input, destFile);
     } catch (Exception ex) {
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 293ece5bf..141741ac5 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -1,19 +1,14 @@
 package net.snowflake.ingest;
 
-import static net.snowflake.ingest.utils.Constants.ACCOUNT;
-import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
-import static net.snowflake.ingest.utils.Constants.CONNECT_STRING;
-import static net.snowflake.ingest.utils.Constants.DATABASE;
-import static net.snowflake.ingest.utils.Constants.HOST;
-import static net.snowflake.ingest.utils.Constants.PORT;
-import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
-import static net.snowflake.ingest.utils.Constants.ROLE;
-import static net.snowflake.ingest.utils.Constants.SCHEMA;
-import static net.snowflake.ingest.utils.Constants.SCHEME;
-import static net.snowflake.ingest.utils.Constants.SSL;
-import static net.snowflake.ingest.utils.Constants.USER;
-import static net.snowflake.ingest.utils.Constants.WAREHOUSE;
-import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION;
+import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
+import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
+import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
+import net.snowflake.ingest.streaming.InsertValidationResponse;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.commons.codec.binary.Base64;
+import org.junit.Assert;
 
 import java.io.IOException;
 import java.math.BigDecimal;
@@ -38,19 +33,25 @@
 import java.util.Properties;
 import java.util.Random;
 import java.util.function.Supplier;
-import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
-import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
-import net.snowflake.ingest.streaming.InsertValidationResponse;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.commons.codec.binary.Base64;
-import org.junit.Assert;
+
+import static net.snowflake.ingest.utils.Constants.ACCOUNT;
+import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
+import static net.snowflake.ingest.utils.Constants.CONNECT_STRING;
+import static net.snowflake.ingest.utils.Constants.DATABASE;
+import static net.snowflake.ingest.utils.Constants.HOST;
+import static net.snowflake.ingest.utils.Constants.PORT;
+import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
+import static net.snowflake.ingest.utils.Constants.ROLE;
+import static net.snowflake.ingest.utils.Constants.SCHEMA;
+import static net.snowflake.ingest.utils.Constants.SCHEME;
+import static net.snowflake.ingest.utils.Constants.SSL;
+import static net.snowflake.ingest.utils.Constants.USER;
+import static net.snowflake.ingest.utils.Constants.WAREHOUSE;
+import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION;
 
 public class TestUtils {
   // profile path, follow readme for the format
-  private static final String PROFILE_PATH = "profile.json";
+  private static final String PROFILE_PATH = "profile.json.example";
 
   private static final ObjectMapper mapper = new ObjectMapper();
 

From a3c7a6245b22b5ef7cf2de7d5469a4aafac39655 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Wed, 5 Apr 2023 12:32:47 -0700
Subject: [PATCH 160/356] autoformatting

---
 .../SnowflakeStreamingIngestExample.java      | 11 ++---
 .../streaming/internal/FlushService.java      | 41 ++++++++--------
 .../internal/StreamingIngestStage.java        | 39 ++++++++-------
 .../java/net/snowflake/ingest/TestUtils.java  | 47 +++++++++----------
 4 files changed, 67 insertions(+), 71 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
index e598709b0..52da1e8e4 100644
--- a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
+++ b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
@@ -6,18 +6,17 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import net.snowflake.ingest.streaming.InsertValidationResponse;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
-
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Properties;
+import net.snowflake.ingest.streaming.InsertValidationResponse;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
 
 /**
  * Example on how to use the Streaming Ingest client APIs.
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 84b2cb034..699fa2f12 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -4,21 +4,14 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import com.codahale.metrics.Timer;
-import net.snowflake.client.jdbc.SnowflakeSQLException;
-import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.util.VisibleForTesting;
-import org.apache.arrow.vector.VectorSchemaRoot;
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
+import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
+import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
+import com.codahale.metrics.Timer;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.security.InvalidAlgorithmParameterException;
@@ -42,13 +35,19 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
-import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
-import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import net.snowflake.client.jdbc.SnowflakeSQLException;
+import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.util.VisibleForTesting;
+import org.apache.arrow.vector.VectorSchemaRoot;
 
 /**
  * Responsible for flushing data from client to Snowflake tables. When a flush is triggered, it will
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index ebceda731..d3b285cd5 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -4,6 +4,24 @@
 
 package net.snowflake.ingest.streaming.internal;
 
+import static net.snowflake.client.core.Constants.CLOUD_STORAGE_CREDENTIALS_EXPIRED;
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CLIENT_CONFIGURE;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
+import static net.snowflake.ingest.utils.Constants.CLIENT_CONFIGURE_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
+import static net.snowflake.ingest.utils.HttpUtil.generateProxyPropertiesForJDBC;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import net.snowflake.client.core.OCSPMode;
 import net.snowflake.client.jdbc.SnowflakeFileTransferAgent;
 import net.snowflake.client.jdbc.SnowflakeFileTransferConfig;
@@ -23,25 +41,6 @@
 import net.snowflake.ingest.utils.Utils;
 import org.apache.arrow.util.VisibleForTesting;
 
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Paths;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Properties;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
-
-import static net.snowflake.client.core.Constants.CLOUD_STORAGE_CREDENTIALS_EXPIRED;
-import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CLIENT_CONFIGURE;
-import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
-import static net.snowflake.ingest.utils.Constants.CLIENT_CONFIGURE_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
-import static net.snowflake.ingest.utils.HttpUtil.generateProxyPropertiesForJDBC;
-
 /** Handles uploading files to the Snowflake Streaming Ingest Stage */
 class StreamingIngestStage {
   private static final ObjectMapper mapper = new ObjectMapper();
@@ -397,7 +396,7 @@ void putLocal(String fullFilePath, byte[] data) {
 
     InputStream input = new ByteArrayInputStream(data);
     try {
-      String stageLocation =  this.fileTransferMetadataWithAge.localLocation; 
+      String stageLocation = this.fileTransferMetadataWithAge.localLocation;
       File destFile = Paths.get(stageLocation, fullFilePath).toFile();
       FileUtils.copyInputStreamToFile(input, destFile);
     } catch (Exception ex) {
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 141741ac5..4d52cd69c 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -1,14 +1,19 @@
 package net.snowflake.ingest;
 
-import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
-import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
-import net.snowflake.ingest.streaming.InsertValidationResponse;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.commons.codec.binary.Base64;
-import org.junit.Assert;
+import static net.snowflake.ingest.utils.Constants.ACCOUNT;
+import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
+import static net.snowflake.ingest.utils.Constants.CONNECT_STRING;
+import static net.snowflake.ingest.utils.Constants.DATABASE;
+import static net.snowflake.ingest.utils.Constants.HOST;
+import static net.snowflake.ingest.utils.Constants.PORT;
+import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
+import static net.snowflake.ingest.utils.Constants.ROLE;
+import static net.snowflake.ingest.utils.Constants.SCHEMA;
+import static net.snowflake.ingest.utils.Constants.SCHEME;
+import static net.snowflake.ingest.utils.Constants.SSL;
+import static net.snowflake.ingest.utils.Constants.USER;
+import static net.snowflake.ingest.utils.Constants.WAREHOUSE;
+import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION;
 
 import java.io.IOException;
 import java.math.BigDecimal;
@@ -33,21 +38,15 @@
 import java.util.Properties;
 import java.util.Random;
 import java.util.function.Supplier;
-
-import static net.snowflake.ingest.utils.Constants.ACCOUNT;
-import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
-import static net.snowflake.ingest.utils.Constants.CONNECT_STRING;
-import static net.snowflake.ingest.utils.Constants.DATABASE;
-import static net.snowflake.ingest.utils.Constants.HOST;
-import static net.snowflake.ingest.utils.Constants.PORT;
-import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
-import static net.snowflake.ingest.utils.Constants.ROLE;
-import static net.snowflake.ingest.utils.Constants.SCHEMA;
-import static net.snowflake.ingest.utils.Constants.SCHEME;
-import static net.snowflake.ingest.utils.Constants.SSL;
-import static net.snowflake.ingest.utils.Constants.USER;
-import static net.snowflake.ingest.utils.Constants.WAREHOUSE;
-import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION;
+import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
+import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
+import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
+import net.snowflake.ingest.streaming.InsertValidationResponse;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.commons.codec.binary.Base64;
+import org.junit.Assert;
 
 public class TestUtils {
   // profile path, follow readme for the format

From 79337a6acfde49324e359a1c2fbaffb7e589fd82 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Tue, 11 Apr 2023 12:25:54 -0700
Subject: [PATCH 161/356] bloblatencies obj

---
 .../streaming/internal/BlobBuilder.java       | 43 ++++----
 .../streaming/internal/BlobLatencies.java     | 69 +++++++++++++
 .../streaming/internal/BlobMetadata.java      | 40 +++-----
 .../streaming/internal/FlushService.java      | 57 +++++------
 .../streaming/internal/RegisterService.java   | 20 ++--
 ...nowflakeStreamingIngestClientInternal.java | 98 +++++++++----------
 .../streaming/internal/FlushServiceTest.java  | 62 ++++++------
 .../internal/RegisterServiceTest.java         | 17 ++--
 .../SnowflakeStreamingIngestClientTest.java   | 85 ++++++++--------
 9 files changed, 275 insertions(+), 216 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
index 235f6c814..745f1ed9c 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
@@ -4,18 +4,17 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
-import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
-import static net.snowflake.ingest.utils.Utils.toByteArray;
-
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.Cryptor;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import org.apache.commons.codec.binary.Hex;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.security.InvalidAlgorithmParameterException;
@@ -26,14 +25,16 @@
 import java.util.List;
 import java.util.zip.CRC32;
 import java.util.zip.GZIPOutputStream;
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.Cryptor;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import org.apache.commons.codec.binary.Hex;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
+import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
+import static net.snowflake.ingest.utils.Utils.toByteArray;
 
 /**
  * Build a single blob file that contains file header plus data. The header will be a
@@ -157,7 +158,7 @@ static <T> Blob constructBlobAndMetadata(
     // Build blob file bytes
     byte[] blobBytes =
         buildBlob(chunksMetadataList, chunksDataList, crc.getValue(), curDataSize, bdecVersion);
-    return new Blob(blobBytes, chunksMetadataList);
+    return new Blob(blobBytes, chunksMetadataList, new BlobLatencies());
   }
 
   /**
@@ -327,10 +328,12 @@ static String computeMD5(byte[] data, int length) throws NoSuchAlgorithmExceptio
   static class Blob {
     final byte[] blobBytes;
     final List<ChunkMetadata> chunksMetadataList;
+    final BlobLatencies blobLatencies;
 
-    Blob(byte[] blobBytes, List<ChunkMetadata> chunksMetadataList) {
+    Blob(byte[] blobBytes, List<ChunkMetadata> chunksMetadataList, BlobLatencies blobLatencies) {
       this.blobBytes = blobBytes;
       this.chunksMetadataList = chunksMetadataList;
+      this.blobLatencies = blobLatencies;
     }
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
new file mode 100644
index 000000000..7d2ffd189
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import com.codahale.metrics.Timer;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Latency information for a blob
+ */
+class BlobLatencies {
+    public static final long DEFAULT_BLOB_LATENCY = -1;
+
+    private long buildLatencyMs;
+    private long uploadLatencyMs;
+
+    private long flushStartTimestamp;
+    private long registerStartTimestamp;
+
+    public BlobLatencies() {
+        this.buildLatencyMs = DEFAULT_BLOB_LATENCY;
+        this.uploadLatencyMs = DEFAULT_BLOB_LATENCY;
+
+        this.flushStartTimestamp = DEFAULT_BLOB_LATENCY;
+        this.registerStartTimestamp = DEFAULT_BLOB_LATENCY;
+    }
+
+    @JsonProperty("build_latency_ms")
+    long getBuildLatencyMs() {
+        return this.buildLatencyMs;
+    }
+
+    @JsonProperty("upload_latency_ms")
+    long getUploadLatencyMs() {
+        return this.uploadLatencyMs;
+    }
+
+    @JsonProperty("flush_start_timestamp")
+    long getFlushStartTimestamp() {
+        return this.flushStartTimestamp;
+    }
+
+    @JsonProperty("register_start_timestamp")
+    long getRegisterStartTimestamp() {
+        return this.registerStartTimestamp;
+    }
+
+    void setBuildLatencyMs(Timer.Context buildLatencyContext) {
+        if (buildLatencyContext != null) {
+            this.buildLatencyMs = buildLatencyContext.stop();
+        }
+    }
+
+    void setUploadLatencyMs(Timer.Context uploadLatencyContext) {
+        if (uploadLatencyContext != null) {
+            this.uploadLatencyMs = uploadLatencyContext.stop();
+        }
+    }
+
+    void setFlushStartTimestamp(long flushStartTimestamp) {
+        this.flushStartTimestamp = flushStartTimestamp;
+    }
+
+    void setRegisterStartTimestamp(long registerStartTimestamp) {
+        this.registerStartTimestamp = registerStartTimestamp;
+    }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index 688c21e69..15fce98c4 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -6,32 +6,27 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import java.util.List;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ParameterProvider;
 
+import java.util.List;
+
 /** Metadata for a blob that sends to Snowflake as part of the register blob request */
 class BlobMetadata {
-  public static final long DEFAULT_BLOB_LATENCY = -1;
-
   private final String path;
   private final String md5;
   private final Constants.BdecVersion bdecVersion;
   private final List<ChunkMetadata> chunks;
-
-  // latencies
-  private final long buildLatencyMs;
-  private final long uploadLatencyMs;
+  private final BlobLatencies blobLatencies;
 
   // used for testing only
-  BlobMetadata(String path, String md5, List<ChunkMetadata> chunks) {
+  BlobMetadata(String path, String md5, List<ChunkMetadata> chunks, BlobLatencies blobLatencies) {
     this(
         path,
         md5,
         ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT,
         chunks,
-        DEFAULT_BLOB_LATENCY,
-        DEFAULT_BLOB_LATENCY);
+        blobLatencies);
   }
 
   BlobMetadata(
@@ -39,15 +34,12 @@ class BlobMetadata {
       String md5,
       Constants.BdecVersion bdecVersion,
       List<ChunkMetadata> chunks,
-      long buildLatencyMs,
-      long uploadLatencyMs) {
+      BlobLatencies blobLatencies) {
     this.path = path;
     this.md5 = md5;
     this.bdecVersion = bdecVersion;
     this.chunks = chunks;
-
-    this.buildLatencyMs = buildLatencyMs;
-    this.uploadLatencyMs = uploadLatencyMs;
+    this.blobLatencies = blobLatencies;
   }
 
   @JsonIgnore
@@ -75,15 +67,8 @@ byte getVersionByte() {
     return bdecVersion.toByte();
   }
 
-  @JsonProperty("build_latency_ms")
-  long getBuildLatencyMs() {
-    return this.buildLatencyMs;
-  }
-
-  @JsonProperty("upload_latency_ms")
-  long getUploadLatencyMs() {
-    return this.uploadLatencyMs;
-  }
+  @JsonProperty("blob_latencies")
+  BlobLatencies getBlobLatencies() { return this.blobLatencies; }
 
   /** Create {@link BlobMetadata}. */
   static BlobMetadata createBlobMetadata(
@@ -91,8 +76,9 @@ static BlobMetadata createBlobMetadata(
       String md5,
       Constants.BdecVersion bdecVersion,
       List<ChunkMetadata> chunks,
-      long buildLatencyMs,
-      long uploadLatencyMs) {
-    return new BlobMetadata(path, md5, bdecVersion, chunks, buildLatencyMs, uploadLatencyMs);
+      BlobLatencies blobLatencies) {
+    return new BlobMetadata(path, md5, bdecVersion, chunks, blobLatencies);
   }
+
+
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 699fa2f12..61a4aef85 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -4,14 +4,21 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
-import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
-import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
-
 import com.codahale.metrics.Timer;
+import net.snowflake.client.jdbc.SnowflakeSQLException;
+import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.util.VisibleForTesting;
+import org.apache.arrow.vector.VectorSchemaRoot;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.security.InvalidAlgorithmParameterException;
@@ -35,19 +42,13 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import net.snowflake.client.jdbc.SnowflakeSQLException;
-import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.util.VisibleForTesting;
-import org.apache.arrow.vector.VectorSchemaRoot;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
+import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
+import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
 /**
  * Responsible for flushing data from client to Snowflake tables. When a flush is triggered, it will
@@ -491,10 +492,9 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
     // Construct the blob along with the metadata of the blob
     BlobBuilder.Blob blob = BlobBuilder.constructBlobAndMetadata(filePath, blobData, bdecVersion);
 
-    return buildContext != null
-        ? upload(filePath, blob.blobBytes, blob.chunksMetadataList, buildContext.stop())
-        : upload(
-            filePath, blob.blobBytes, blob.chunksMetadataList, BlobMetadata.DEFAULT_BLOB_LATENCY);
+    blob.blobLatencies.setBuildLatencyMs(buildContext);
+
+    return upload(filePath, blob.blobBytes, blob.chunksMetadataList, blob.blobLatencies);
   }
 
   /**
@@ -503,17 +503,15 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
    * @param filePath full path of the blob file
    * @param blob blob data
    * @param metadata a list of chunk metadata
-   * @param buildLatencyMs the build latency for the blob
    * @return BlobMetadata object used to create the register blob request
    */
   BlobMetadata upload(
-      String filePath, byte[] blob, List<ChunkMetadata> metadata, long buildLatencyMs)
+      String filePath, byte[] blob, List<ChunkMetadata> metadata, BlobLatencies blobLatencies)
       throws NoSuchAlgorithmException {
     logger.logInfo("Start uploading file={}, size={}", filePath, blob.length);
     long startTime = System.currentTimeMillis();
 
     Timer.Context uploadContext = Utils.createTimerContext(this.owningClient.uploadLatency);
-    long uploadLatencyMs = BlobMetadata.DEFAULT_BLOB_LATENCY;
 
     this.targetStage.put(filePath, blob);
 
@@ -524,7 +522,7 @@ BlobMetadata upload(
         System.currentTimeMillis() - startTime);
 
     if (uploadContext != null) {
-      uploadLatencyMs = uploadContext.stop();
+      blobLatencies.setUploadLatencyMs(uploadContext);
       this.owningClient.uploadThroughput.mark(blob.length);
       this.owningClient.blobSizeHistogram.update(blob.length);
       this.owningClient.blobRowCountHistogram.update(
@@ -536,8 +534,7 @@ BlobMetadata upload(
         BlobBuilder.computeMD5(blob),
         bdecVersion,
         metadata,
-        buildLatencyMs,
-        uploadLatencyMs);
+        blobLatencies);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index b9eed9b6d..d4e33d50b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -4,10 +4,11 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
-
 import com.codahale.metrics.Timer;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.Utils;
+
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -17,9 +18,9 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.Utils;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
 /**
  * Register one or more blobs to the targeted Snowflake table, it will be done using the dedicated
@@ -198,8 +199,13 @@ List<FlushService.BlobData<T>> registerBlobs(
             Timer.Context registerContext =
                 Utils.createTimerContext(this.owningClient.registerLatency);
 
+            for (BlobMetadata blobMetadata : blobs) {
+              blobMetadata.getBlobLatencies().setRegisterStartTimestamp(System.currentTimeMillis());
+              blobMetadata.getBlobLatencies().setFlushStartTimestamp(System.currentTimeMillis());
+            }
+
             // Register the blobs, and invalidate any channels that return a failure status code
-            this.owningClient.registerBlobs(blobs, flushJobStartTime);
+            this.owningClient.registerBlobs(blobs);
 
             if (registerContext != null) {
               registerContext.stop();
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 47f92392f..78e7c2ff3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -4,28 +4,6 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CHANNEL_STATUS;
-import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_OPEN_CHANNEL;
-import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_REGISTER_BLOB;
-import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
-import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.sleepForRetry;
-import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.COMMIT_MAX_RETRY_COUNT;
-import static net.snowflake.ingest.utils.Constants.COMMIT_RETRY_INTERVAL_IN_MS;
-import static net.snowflake.ingest.utils.Constants.ENABLE_TELEMETRY_TO_SF;
-import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
-import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
-import static net.snowflake.ingest.utils.Constants.OPEN_CHANNEL_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
-import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JMX_METRIC_PREFIX;
-import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JVM_MEMORY_AND_THREAD_METRICS_REGISTRY;
-import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_SHARED_METRICS_REGISTRY;
-import static net.snowflake.ingest.utils.Constants.STREAMING_INGEST_TELEMETRY_UPLOAD_INTERVAL_IN_SEC;
-import static net.snowflake.ingest.utils.Constants.USER;
-
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.MetricFilter;
@@ -37,6 +15,26 @@
 import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
 import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
+import net.snowflake.ingest.connection.IngestResponseException;
+import net.snowflake.ingest.connection.RequestBuilder;
+import net.snowflake.ingest.connection.TelemetryService;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.HttpUtil;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.ParameterProvider;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.SnowflakeURL;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
 import java.io.IOException;
 import java.security.KeyPair;
 import java.security.NoSuchAlgorithmException;
@@ -56,25 +54,28 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
-import javax.management.MalformedObjectNameException;
-import javax.management.ObjectName;
-import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
-import net.snowflake.ingest.connection.IngestResponseException;
-import net.snowflake.ingest.connection.RequestBuilder;
-import net.snowflake.ingest.connection.TelemetryService;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.HttpUtil;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.ParameterProvider;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.SnowflakeURL;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
+
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CHANNEL_STATUS;
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_OPEN_CHANNEL;
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_REGISTER_BLOB;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.sleepForRetry;
+import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.COMMIT_MAX_RETRY_COUNT;
+import static net.snowflake.ingest.utils.Constants.COMMIT_RETRY_INTERVAL_IN_MS;
+import static net.snowflake.ingest.utils.Constants.ENABLE_TELEMETRY_TO_SF;
+import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
+import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
+import static net.snowflake.ingest.utils.Constants.OPEN_CHANNEL_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
+import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JMX_METRIC_PREFIX;
+import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JVM_MEMORY_AND_THREAD_METRICS_REGISTRY;
+import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_SHARED_METRICS_REGISTRY;
+import static net.snowflake.ingest.utils.Constants.STREAMING_INGEST_TELEMETRY_UPLOAD_INTERVAL_IN_SEC;
+import static net.snowflake.ingest.utils.Constants.USER;
 
 /**
  * The first version of implementation for SnowflakeStreamingIngestClient. The client internally
@@ -404,10 +405,9 @@ ChannelsStatusResponse getChannelsStatus(
    * Register the uploaded blobs to a Snowflake table
    *
    * @param blobs list of uploaded blobs
-   * @param flushJobStartTime When the flush job began
    */
-  void registerBlobs(List<BlobMetadata> blobs, long flushJobStartTime) {
-    this.registerBlobs(blobs, 0, flushJobStartTime);
+  void registerBlobs(List<BlobMetadata> blobs) {
+    this.registerBlobs(blobs, 0);
   }
 
   /**
@@ -415,9 +415,8 @@ void registerBlobs(List<BlobMetadata> blobs, long flushJobStartTime) {
    *
    * @param blobs list of uploaded blobs
    * @param executionCount Number of times this call has been attempted, used to track retries
-   * @param flushJobStartTime When the flush job began
    */
-  void registerBlobs(List<BlobMetadata> blobs, final int executionCount, long flushJobStartTime) {
+  void registerBlobs(List<BlobMetadata> blobs, final int executionCount) {
     logger.logInfo(
         "Register blob request preparing for blob={}, client={}, executionCount={}",
         blobs.stream().map(BlobMetadata::getPath).collect(Collectors.toList()),
@@ -431,8 +430,6 @@ void registerBlobs(List<BlobMetadata> blobs, final int executionCount, long flus
           "request_id", this.flushService.getClientPrefix() + "_" + counter.getAndIncrement());
       payload.put("blobs", blobs);
       payload.put("role", this.role);
-      payload.put("flush_start_timestamp", flushJobStartTime);
-      payload.put("register_start_timestamp", System.currentTimeMillis());
 
       response =
           executeWithRetries(
@@ -524,7 +521,7 @@ void registerBlobs(List<BlobMetadata> blobs, final int executionCount, long flus
         throw new SFException(ErrorCode.INTERNAL_ERROR, "Failed to retry queue full chunks");
       }
       sleepForRetry(executionCount);
-      this.registerBlobs(retryBlobs, executionCount + 1, flushJobStartTime);
+      this.registerBlobs(retryBlobs, executionCount + 1);
     }
   }
 
@@ -577,8 +574,7 @@ List<BlobMetadata> getRetryBlobs(
                     blobMetadata.getMD5(),
                     blobMetadata.getVersion(),
                     relevantChunks,
-                    blobMetadata.getBuildLatencyMs(),
-                    blobMetadata.getUploadLatencyMs()));
+                    blobMetadata.getBlobLatencies()));
           }
         });
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 0577ac107..d68570ecc 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -1,14 +1,28 @@
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
-import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
-
 import com.fasterxml.jackson.databind.ObjectMapper;
+import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.Cryptor;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.ParameterProvider;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.ArgumentCaptor;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.math.BigInteger;
@@ -32,27 +46,14 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TimeZone;
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.Cryptor;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.ParameterProvider;
-import net.snowflake.ingest.utils.SFException;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.mockito.ArgumentCaptor;
-import org.mockito.ArgumentMatchers;
-import org.mockito.Mockito;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
+import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
 
 @RunWith(Parameterized.class)
 public class FlushServiceTest {
@@ -603,8 +604,7 @@ public void testBuildAndUpload() throws Exception {
         .upload(
             nameCaptor.capture(),
             blobCaptor.capture(),
-            metadataCaptor.capture(),
-            ArgumentMatchers.anyLong());
+            metadataCaptor.capture(), ArgumentMatchers.any());
     Assert.assertEquals("file_name", nameCaptor.getValue());
 
     ChunkMetadata metadataResult = metadataCaptor.getValue().get(0);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
index 16cac7345..e543229ba 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
@@ -1,6 +1,9 @@
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
+import net.snowflake.ingest.utils.Pair;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -9,10 +12,8 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import net.snowflake.ingest.utils.Pair;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
 
 public class RegisterServiceTest {
 
@@ -23,7 +24,7 @@ public void testRegisterService() {
     Pair<FlushService.BlobData<StubChunkData>, CompletableFuture<BlobMetadata>> blobFuture =
         new Pair<>(
             new FlushService.BlobData<>("test", null),
-            CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null)));
+            CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null, null)));
     rs.addBlobs(Collections.singletonList(blobFuture));
     Assert.assertEquals(1, rs.getBlobsList().size());
     List<FlushService.BlobData<StubChunkData>> errorBlobs =
@@ -50,7 +51,7 @@ public void testRegisterServiceTimeoutException() throws Exception {
     Pair<FlushService.BlobData<StubChunkData>, CompletableFuture<BlobMetadata>> blobFuture1 =
         new Pair<>(
             new FlushService.BlobData<>("success", new ArrayList<>()),
-            CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null)));
+            CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null, null)));
     CompletableFuture future = new CompletableFuture();
     future.completeExceptionally(new TimeoutException());
     Pair<FlushService.BlobData<StubChunkData>, CompletableFuture<BlobMetadata>> blobFuture2 =
@@ -79,7 +80,7 @@ public void testRegisterServiceTimeoutException_testRetries() throws Exception {
     Pair<FlushService.BlobData<StubChunkData>, CompletableFuture<BlobMetadata>> blobFuture1 =
         new Pair<>(
             new FlushService.BlobData<>("success", new ArrayList<>()),
-            CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null)));
+            CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null, null)));
     CompletableFuture future = new CompletableFuture();
     future.thenRunAsync(
         () -> {
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index 08c30edcc..56dc492a3 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -1,37 +1,8 @@
 package net.snowflake.ingest.streaming.internal;
 
-import static java.time.ZoneOffset.UTC;
-import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
-import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
-import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
-import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
-import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
-import static net.snowflake.ingest.utils.Constants.ROLE;
-import static net.snowflake.ingest.utils.Constants.USER;
-import static net.snowflake.ingest.utils.ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS;
-
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.PropertyAccessor;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import java.io.IOException;
-import java.io.StringWriter;
-import java.security.KeyPair;
-import java.security.PrivateKey;
-import java.security.Security;
-import java.time.ZoneOffset;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.stream.Collectors;
 import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils;
 import net.snowflake.client.jdbc.internal.apache.http.HttpEntity;
 import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders;
@@ -66,6 +37,36 @@
 import org.junit.Test;
 import org.mockito.Mockito;
 
+import java.io.IOException;
+import java.io.StringWriter;
+import java.security.KeyPair;
+import java.security.PrivateKey;
+import java.security.Security;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+
+import static java.time.ZoneOffset.UTC;
+import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
+import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
+import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
+import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
+import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
+import static net.snowflake.ingest.utils.Constants.ROLE;
+import static net.snowflake.ingest.utils.Constants.USER;
+import static net.snowflake.ingest.utils.ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS;
+
 public class SnowflakeStreamingIngestClientTest {
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
@@ -487,7 +488,7 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
 
     List<BlobMetadata> blobs =
         Collections.singletonList(
-            new BlobMetadata("path", "md5", Collections.singletonList(chunkMetadata)));
+            new BlobMetadata("path", "md5", Collections.singletonList(chunkMetadata), null));
 
     Map<Object, Object> payload = new HashMap<>();
     payload.put("request_id", null);
@@ -587,8 +588,8 @@ private Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> getRetryBlobMetadata(
     chunks1.add(chunkMetadata1);
     chunks1.add(chunkMetadata2);
     chunks2.add(chunkMetadata3);
-    blobs.add(new BlobMetadata("path1", "md51", chunks1));
-    blobs.add(new BlobMetadata("path2", "md52", chunks2));
+    blobs.add(new BlobMetadata("path1", "md51", chunks1, null));
+    blobs.add(new BlobMetadata("path2", "md52", chunks2, null));
 
     List<ChannelRegisterStatus> channelRegisterStatuses = new ArrayList<>();
     ChannelRegisterStatus status1 = new ChannelRegisterStatus();
@@ -673,8 +674,8 @@ public void testRegisterBlobErrorResponse() throws Exception {
 
     try {
       List<BlobMetadata> blobs =
-          Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
-      client.registerBlobs(blobs, System.currentTimeMillis());
+          Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>(), null));
+      client.registerBlobs(blobs);
       Assert.fail("Register blob should fail on 404 error");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.REGISTER_BLOB_FAILURE.getMessageCode(), e.getVendorCode());
@@ -720,8 +721,8 @@ public void testRegisterBlobSnowflakeInternalErrorResponse() throws Exception {
 
     try {
       List<BlobMetadata> blobs =
-          Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
-      client.registerBlobs(blobs, System.currentTimeMillis());
+          Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>(), null));
+      client.registerBlobs(blobs);
       Assert.fail("Register blob should fail on SF internal error");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.REGISTER_BLOB_FAILURE.getMessageCode(), e.getVendorCode());
@@ -775,8 +776,8 @@ public void testRegisterBlobSuccessResponse() throws Exception {
             null);
 
     List<BlobMetadata> blobs =
-        Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
-    client.registerBlobs(blobs, System.currentTimeMillis());
+        Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>(), null));
+    client.registerBlobs(blobs);
   }
 
   @Test
@@ -862,7 +863,7 @@ public void testRegisterBlobsRetries() throws Exception {
     client.getChannelCache().addChannel(channel2);
     client.getChannelCache().addChannel(channel3);
     client.getChannelCache().addChannel(channel4);
-    client.registerBlobs(blobs, System.currentTimeMillis());
+    client.registerBlobs(blobs);
     Mockito.verify(requestBuilder, Mockito.times(MAX_STREAMING_INGEST_API_CHANNEL_RETRY + 1))
         .generateStreamingIngestPostRequest(Mockito.anyString(), Mockito.any(), Mockito.any());
     Assert.assertFalse(channel1.isValid());
@@ -978,7 +979,7 @@ public void testRegisterBlobsRetriesSucceeds() throws Exception {
     client.getChannelCache().addChannel(channel3);
     client.getChannelCache().addChannel(channel4);
 
-    client.registerBlobs(blobs, System.currentTimeMillis());
+    client.registerBlobs(blobs);
     Mockito.verify(requestBuilder, Mockito.times(2))
         .generateStreamingIngestPostRequest(Mockito.anyString(), Mockito.any(), Mockito.any());
     Assert.assertTrue(channel1.isValid());
@@ -1082,8 +1083,8 @@ public void testRegisterBlobResponseWithInvalidChannel() throws Exception {
     Assert.assertTrue(channel2.isValid());
 
     List<BlobMetadata> blobs =
-        Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
-    client.registerBlobs(blobs, System.currentTimeMillis());
+        Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>(), null));
+    client.registerBlobs(blobs);
 
     // Channel2 should be invalidated now
     Assert.assertTrue(channel1.isValid());

From 5c93b6b875940ff05aa2304120622230190cb9f3 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Tue, 11 Apr 2023 12:25:56 -0700
Subject: [PATCH 162/356] autoformatting

---
 .../streaming/internal/BlobBuilder.java       | 37 ++++----
 .../streaming/internal/BlobLatencies.java     | 86 +++++++++----------
 .../streaming/internal/BlobMetadata.java      | 16 ++--
 .../streaming/internal/FlushService.java      | 47 +++++-----
 .../streaming/internal/RegisterService.java   | 13 ++-
 ...nowflakeStreamingIngestClientInternal.java | 83 +++++++++---------
 .../streaming/internal/FlushServiceTest.java  | 62 ++++++-------
 .../internal/RegisterServiceTest.java         | 11 ++-
 .../SnowflakeStreamingIngestClientTest.java   | 59 +++++++------
 9 files changed, 198 insertions(+), 216 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
index 745f1ed9c..e4f0b807e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
@@ -4,17 +4,18 @@
 
 package net.snowflake.ingest.streaming.internal;
 
+import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
+import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
+import static net.snowflake.ingest.utils.Utils.toByteArray;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.Cryptor;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import org.apache.commons.codec.binary.Hex;
-
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.security.InvalidAlgorithmParameterException;
@@ -25,16 +26,14 @@
 import java.util.List;
 import java.util.zip.CRC32;
 import java.util.zip.GZIPOutputStream;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
-import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
-import static net.snowflake.ingest.utils.Utils.toByteArray;
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.Cryptor;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import org.apache.commons.codec.binary.Hex;
 
 /**
  * Build a single blob file that contains file header plus data. The header will be a
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
index 7d2ffd189..171fef084 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
@@ -7,63 +7,61 @@
 import com.codahale.metrics.Timer;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
-/**
- * Latency information for a blob
- */
+/** Latency information for a blob */
 class BlobLatencies {
-    public static final long DEFAULT_BLOB_LATENCY = -1;
+  public static final long DEFAULT_BLOB_LATENCY = -1;
 
-    private long buildLatencyMs;
-    private long uploadLatencyMs;
+  private long buildLatencyMs;
+  private long uploadLatencyMs;
 
-    private long flushStartTimestamp;
-    private long registerStartTimestamp;
+  private long flushStartTimestamp;
+  private long registerStartTimestamp;
 
-    public BlobLatencies() {
-        this.buildLatencyMs = DEFAULT_BLOB_LATENCY;
-        this.uploadLatencyMs = DEFAULT_BLOB_LATENCY;
+  public BlobLatencies() {
+    this.buildLatencyMs = DEFAULT_BLOB_LATENCY;
+    this.uploadLatencyMs = DEFAULT_BLOB_LATENCY;
 
-        this.flushStartTimestamp = DEFAULT_BLOB_LATENCY;
-        this.registerStartTimestamp = DEFAULT_BLOB_LATENCY;
-    }
+    this.flushStartTimestamp = DEFAULT_BLOB_LATENCY;
+    this.registerStartTimestamp = DEFAULT_BLOB_LATENCY;
+  }
 
-    @JsonProperty("build_latency_ms")
-    long getBuildLatencyMs() {
-        return this.buildLatencyMs;
-    }
+  @JsonProperty("build_latency_ms")
+  long getBuildLatencyMs() {
+    return this.buildLatencyMs;
+  }
 
-    @JsonProperty("upload_latency_ms")
-    long getUploadLatencyMs() {
-        return this.uploadLatencyMs;
-    }
+  @JsonProperty("upload_latency_ms")
+  long getUploadLatencyMs() {
+    return this.uploadLatencyMs;
+  }
 
-    @JsonProperty("flush_start_timestamp")
-    long getFlushStartTimestamp() {
-        return this.flushStartTimestamp;
-    }
+  @JsonProperty("flush_start_timestamp")
+  long getFlushStartTimestamp() {
+    return this.flushStartTimestamp;
+  }
 
-    @JsonProperty("register_start_timestamp")
-    long getRegisterStartTimestamp() {
-        return this.registerStartTimestamp;
-    }
+  @JsonProperty("register_start_timestamp")
+  long getRegisterStartTimestamp() {
+    return this.registerStartTimestamp;
+  }
 
-    void setBuildLatencyMs(Timer.Context buildLatencyContext) {
-        if (buildLatencyContext != null) {
-            this.buildLatencyMs = buildLatencyContext.stop();
-        }
+  void setBuildLatencyMs(Timer.Context buildLatencyContext) {
+    if (buildLatencyContext != null) {
+      this.buildLatencyMs = buildLatencyContext.stop();
     }
+  }
 
-    void setUploadLatencyMs(Timer.Context uploadLatencyContext) {
-        if (uploadLatencyContext != null) {
-            this.uploadLatencyMs = uploadLatencyContext.stop();
-        }
+  void setUploadLatencyMs(Timer.Context uploadLatencyContext) {
+    if (uploadLatencyContext != null) {
+      this.uploadLatencyMs = uploadLatencyContext.stop();
     }
+  }
 
-    void setFlushStartTimestamp(long flushStartTimestamp) {
-        this.flushStartTimestamp = flushStartTimestamp;
-    }
+  void setFlushStartTimestamp(long flushStartTimestamp) {
+    this.flushStartTimestamp = flushStartTimestamp;
+  }
 
-    void setRegisterStartTimestamp(long registerStartTimestamp) {
-        this.registerStartTimestamp = registerStartTimestamp;
-    }
+  void setRegisterStartTimestamp(long registerStartTimestamp) {
+    this.registerStartTimestamp = registerStartTimestamp;
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index 15fce98c4..f7d44ae05 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -6,11 +6,10 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ParameterProvider;
 
-import java.util.List;
-
 /** Metadata for a blob that sends to Snowflake as part of the register blob request */
 class BlobMetadata {
   private final String path;
@@ -21,12 +20,7 @@ class BlobMetadata {
 
   // used for testing only
   BlobMetadata(String path, String md5, List<ChunkMetadata> chunks, BlobLatencies blobLatencies) {
-    this(
-        path,
-        md5,
-        ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT,
-        chunks,
-        blobLatencies);
+    this(path, md5, ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT, chunks, blobLatencies);
   }
 
   BlobMetadata(
@@ -68,7 +62,9 @@ byte getVersionByte() {
   }
 
   @JsonProperty("blob_latencies")
-  BlobLatencies getBlobLatencies() { return this.blobLatencies; }
+  BlobLatencies getBlobLatencies() {
+    return this.blobLatencies;
+  }
 
   /** Create {@link BlobMetadata}. */
   static BlobMetadata createBlobMetadata(
@@ -79,6 +75,4 @@ static BlobMetadata createBlobMetadata(
       BlobLatencies blobLatencies) {
     return new BlobMetadata(path, md5, bdecVersion, chunks, blobLatencies);
   }
-
-
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 61a4aef85..c90949b08 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -4,21 +4,14 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import com.codahale.metrics.Timer;
-import net.snowflake.client.jdbc.SnowflakeSQLException;
-import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.util.VisibleForTesting;
-import org.apache.arrow.vector.VectorSchemaRoot;
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
+import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
+import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
+import com.codahale.metrics.Timer;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.security.InvalidAlgorithmParameterException;
@@ -42,13 +35,19 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
-import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
-import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import net.snowflake.client.jdbc.SnowflakeSQLException;
+import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.util.VisibleForTesting;
+import org.apache.arrow.vector.VectorSchemaRoot;
 
 /**
  * Responsible for flushing data from client to Snowflake tables. When a flush is triggered, it will
@@ -530,11 +529,7 @@ BlobMetadata upload(
     }
 
     return BlobMetadata.createBlobMetadata(
-        filePath,
-        BlobBuilder.computeMD5(blob),
-        bdecVersion,
-        metadata,
-        blobLatencies);
+        filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata, blobLatencies);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index d4e33d50b..151666656 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -4,11 +4,10 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import com.codahale.metrics.Timer;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.Utils;
+import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
+import com.codahale.metrics.Timer;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -18,9 +17,9 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.Utils;
 
 /**
  * Register one or more blobs to the targeted Snowflake table, it will be done using the dedicated
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 78e7c2ff3..7526b2117 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -4,6 +4,28 @@
 
 package net.snowflake.ingest.streaming.internal;
 
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CHANNEL_STATUS;
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_OPEN_CHANNEL;
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_REGISTER_BLOB;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.sleepForRetry;
+import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.COMMIT_MAX_RETRY_COUNT;
+import static net.snowflake.ingest.utils.Constants.COMMIT_RETRY_INTERVAL_IN_MS;
+import static net.snowflake.ingest.utils.Constants.ENABLE_TELEMETRY_TO_SF;
+import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
+import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
+import static net.snowflake.ingest.utils.Constants.OPEN_CHANNEL_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
+import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JMX_METRIC_PREFIX;
+import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JVM_MEMORY_AND_THREAD_METRICS_REGISTRY;
+import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_SHARED_METRICS_REGISTRY;
+import static net.snowflake.ingest.utils.Constants.STREAMING_INGEST_TELEMETRY_UPLOAD_INTERVAL_IN_SEC;
+import static net.snowflake.ingest.utils.Constants.USER;
+
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.MetricFilter;
@@ -15,26 +37,6 @@
 import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
 import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
-import net.snowflake.ingest.connection.IngestResponseException;
-import net.snowflake.ingest.connection.RequestBuilder;
-import net.snowflake.ingest.connection.TelemetryService;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.HttpUtil;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.ParameterProvider;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.SnowflakeURL;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
-
-import javax.management.MalformedObjectNameException;
-import javax.management.ObjectName;
 import java.io.IOException;
 import java.security.KeyPair;
 import java.security.NoSuchAlgorithmException;
@@ -54,28 +56,25 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
-
-import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CHANNEL_STATUS;
-import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_OPEN_CHANNEL;
-import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_REGISTER_BLOB;
-import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
-import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.sleepForRetry;
-import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.COMMIT_MAX_RETRY_COUNT;
-import static net.snowflake.ingest.utils.Constants.COMMIT_RETRY_INTERVAL_IN_MS;
-import static net.snowflake.ingest.utils.Constants.ENABLE_TELEMETRY_TO_SF;
-import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
-import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
-import static net.snowflake.ingest.utils.Constants.OPEN_CHANNEL_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
-import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JMX_METRIC_PREFIX;
-import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_JVM_MEMORY_AND_THREAD_METRICS_REGISTRY;
-import static net.snowflake.ingest.utils.Constants.SNOWPIPE_STREAMING_SHARED_METRICS_REGISTRY;
-import static net.snowflake.ingest.utils.Constants.STREAMING_INGEST_TELEMETRY_UPLOAD_INTERVAL_IN_SEC;
-import static net.snowflake.ingest.utils.Constants.USER;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
+import net.snowflake.ingest.connection.IngestResponseException;
+import net.snowflake.ingest.connection.RequestBuilder;
+import net.snowflake.ingest.connection.TelemetryService;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.HttpUtil;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.ParameterProvider;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.SnowflakeURL;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
 
 /**
  * The first version of implementation for SnowflakeStreamingIngestClient. The client internally
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index d68570ecc..cbc27ced0 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -1,28 +1,14 @@
 package net.snowflake.ingest.streaming.internal;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.Cryptor;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.ParameterProvider;
-import net.snowflake.ingest.utils.SFException;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.mockito.ArgumentCaptor;
-import org.mockito.ArgumentMatchers;
-import org.mockito.Mockito;
+import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
+import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
 
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.math.BigInteger;
@@ -46,14 +32,27 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TimeZone;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
-import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.Cryptor;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.ParameterProvider;
+import net.snowflake.ingest.utils.SFException;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.ArgumentCaptor;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
 
 @RunWith(Parameterized.class)
 public class FlushServiceTest {
@@ -604,7 +603,8 @@ public void testBuildAndUpload() throws Exception {
         .upload(
             nameCaptor.capture(),
             blobCaptor.capture(),
-            metadataCaptor.capture(), ArgumentMatchers.any());
+            metadataCaptor.capture(),
+            ArgumentMatchers.any());
     Assert.assertEquals("file_name", nameCaptor.getValue());
 
     ChunkMetadata metadataResult = metadataCaptor.getValue().get(0);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
index e543229ba..d35ccf5bb 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
@@ -1,9 +1,6 @@
 package net.snowflake.ingest.streaming.internal;
 
-import net.snowflake.ingest.utils.Pair;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
+import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -12,8 +9,10 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
+import net.snowflake.ingest.utils.Pair;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
 
 public class RegisterServiceTest {
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index 56dc492a3..e799e1b7c 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -1,8 +1,37 @@
 package net.snowflake.ingest.streaming.internal;
 
+import static java.time.ZoneOffset.UTC;
+import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
+import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
+import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
+import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
+import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
+import static net.snowflake.ingest.utils.Constants.ROLE;
+import static net.snowflake.ingest.utils.Constants.USER;
+import static net.snowflake.ingest.utils.ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS;
+
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.PropertyAccessor;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.security.KeyPair;
+import java.security.PrivateKey;
+import java.security.Security;
+import java.time.ZoneOffset;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
 import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils;
 import net.snowflake.client.jdbc.internal.apache.http.HttpEntity;
 import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders;
@@ -37,36 +66,6 @@
 import org.junit.Test;
 import org.mockito.Mockito;
 
-import java.io.IOException;
-import java.io.StringWriter;
-import java.security.KeyPair;
-import java.security.PrivateKey;
-import java.security.Security;
-import java.time.ZoneOffset;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.stream.Collectors;
-
-import static java.time.ZoneOffset.UTC;
-import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
-import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
-import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
-import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
-import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
-import static net.snowflake.ingest.utils.Constants.ROLE;
-import static net.snowflake.ingest.utils.Constants.USER;
-import static net.snowflake.ingest.utils.ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS;
-
 public class SnowflakeStreamingIngestClientTest {
   private static final ObjectMapper objectMapper = new ObjectMapper();
 

From 83aae5ceaa3eb33f866725636755e4c4ab416fca Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Wed, 5 Apr 2023 14:05:54 -0700
Subject: [PATCH 163/356] personal nits and revert profile.json.example

---
 profile.json.example                          | 22 +++++++++----------
 .../streaming/internal/FlushService.java      |  1 +
 .../java/net/snowflake/ingest/TestUtils.java  |  2 +-
 3 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/profile.json.example b/profile.json.example
index 14007cfd0..46dbd42f0 100644
--- a/profile.json.example
+++ b/profile.json.example
@@ -1,15 +1,15 @@
 {
-  "user": "admin",
-  "url": "http://snowflake.dev.local:8082/",
-  "account": "snowflake",
-  "private_key": "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCsFG3wgtImSV01sl3dbvNotwPTbFiOWz/W1zo/exO840uNyI1VYQ++Dzfxw5Vpp7KwELmPD8xZC0S4FmfYxSJa293qHDYM5sVnH4VYl/Jx5y8rubL9Ye1KQGgW7+uW7O/vgUzlwakMnwVd6Yrr9mPM1HF/E6uTPppi9sTgiylOwZb5CKoctXmZyi3oemotK7r797id+612HSNPxgmrdp6Sj7ckDDABnchTe8JqDTcAlupJApZCUSywZg15LB/lBiGJrGFp0MaxtrvZl1sr+OEqCflKyiRsjvVRnF9YOHu2Msa6FG3/xcUTkNDK4wwiqSgzfhM2qQpwN6nD8IahPpAdAgMBAAECggEABTI8u6WHqQZQHT38naIrEX9tSyYbankQ7XDkXmReDTcj4e0rb2eV7RxSiSdpzJ32xGBM6igE+K9hbNmoWyBg0DOReO9PJoaVcW6/PRShipT7lKtv3zzkyHg3bHHkQFN5T1ctNkbpzurCY7qYqlVvYBEOOFMkvDaIvPJMsaciITGkqtK8nI64BMKH2zR8pZt4ZgIczN5PojXjbhqubWnaIWEle+AOV08BQJoH/63nxZj6uKq8UnER0jPp2Hz/+bHFPv6McYrweo+VFI6OaLTAWk8UKGqoMVkJISaDb/uBiM6vISEJkLKqRE1zeaz+cBWQSy0fCDmgAmGKybq8xpvWdQKBgQDV42p6CESyyriWS7U2iQ+cP/W4NOEfQCS0BZ1ss5rrrDAKK2lxUgNdQiwfmH1V801dNaLosvb+0UtNQQI6kG47BneFvw/1OT/2CE0GqAO7HvY3qIgy/+WaICZEUVV0mE82+irX0Wi0SusBx1qVtkOe/orjU/JdlaRtSVhkYF+GFwKBgQDN9b5Xfv0vZdRTeYpc5Oc2NzhbWhVyEpaOmGfuEEVQbaWL+2lpQyf+MuhUmCOtX3ZEjSuMSr3hYrI5Hg2iwB95wK2fPrCELdD7WpYjdGnzlfO+mDEMM3SlSpphEy7k7PaV2onmrRrWtMNUwPtyiFZ7RRDpmWlrAnm7FeK1Sfnv6wKBgGexJMGcmJGFLg/PDD8wG11ItLtlB267DhAoWLWGLSjsBS4o9al2dSMVHYcDyl+M4Ii7pArkGFJY7QS663Ww53++FtvEUNw1sQh55b3AXm6tB/jbv4vZ+1nJFEQymhc4b87bYUHXx8cAOwZVu1ixT/8YsASdgfZUwa4pzKx8/FJpAoGBAIp77Aok5tSR4ZxnIrOnbhe0NX6nKbhT60viCi+2XQThVOi0mYIfl6qCTFllGsgeYgVh7qAcOWRs2m0xWfXOvNs+xP9IRaP6soPvuvgH9J3Ge+fxqTkM+CeT7A3NukBPXNYR5ZtnVZ58WMKKKAgwyAWGwQJxFsLPHU7APnmrThUVAoGBAM8MUDbaWWITWjQmdx44spukPJ/tt5bPMocarJbb9htPLgqZtpun+uHoBRjbSweJf1jDe/mKTgClR93ZjZG69TnbGQwrKnxZv1Hn1NY7kUnPb+uYs4miGr9tjQ48xAOlcyzfrU1UPXWlWkDFN9GicaNasa11F3npvf8MUBI8d8So",
-  "port": 8082,
-  "host": "snowflake.dev.local",
-  "schema": "PUBLIC",
+  "user": "user name",
+  "url": "https://account_name.snowflakecomputing.com:443",
+  "account": "account_name",
+  "private_key": "PEM Private Key",
+  "port": 443,
+  "host": "account_name.snowflakecomputing.com",
+  "schema": "schema",
   "scheme": "https",
-  "database": "REPLICATION",
-  "connect_string": "jdbc:snowflake://snowflake.dev.local:8082",
+  "database": "database_name",
+  "connect_string": "jdbc:snowflake://account_name.snowflakecomputing.com:443",
   "ssl": "on",
-  "warehouse": "compute_service_warehouse",
-  "role": "accountadmin"
+  "warehouse": "warehouse_name",
+  "role": "role_name"
 }
\ No newline at end of file
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index c90949b08..64f543d52 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -348,6 +348,7 @@ void distributeFlushTasks() {
       List<List<ChannelData<T>>> blobData = new ArrayList<>();
       float totalBufferSizeInBytes = 0F;
       final String filePath = getFilePath(this.targetStage.getClientPrefix());
+
       // Distribute work at table level, split the blob if reaching the blob size limit or the
       // channel has different encryption key ids
       while (itr.hasNext() || !leftoverChannelsDataPerTable.isEmpty()) {
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 4d52cd69c..293ece5bf 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -50,7 +50,7 @@
 
 public class TestUtils {
   // profile path, follow readme for the format
-  private static final String PROFILE_PATH = "profile.json.example";
+  private static final String PROFILE_PATH = "profile.json";
 
   private static final ObjectMapper mapper = new ObjectMapper();
 

From 1e1a7a362996f692f0e782ce36c739f9e7b9c661 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Wed, 5 Apr 2023 15:14:41 -0700
Subject: [PATCH 164/356] build and upload latency test

---
 .../SnowflakeStreamingIngestExample.java      |  3 +-
 .../internal/StreamingIngestStage.java        |  1 +
 .../streaming/internal/FlushServiceTest.java  | 33 ++++++++++++++++---
 3 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
index 52da1e8e4..0a565c97a 100644
--- a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
+++ b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
@@ -24,10 +24,11 @@
  * <p>Please read the README.md file for detailed steps
  */
 public class SnowflakeStreamingIngestExample {
+  // TODO @rcheng - revert this before merge
   // Please follow the example in profile_streaming.json.example to see the required properties, or
   // if you have already set up profile.json with Snowpipe before, all you need is to add the "role"
   // property.
-  private static String PROFILE_PATH = "profile.json.example";
+  private static String PROFILE_PATH = "/home/rcheng/tmp/profile.json.example";
   private static final ObjectMapper mapper = new ObjectMapper();
   private static final String TABLE_NAME = "revi_ingest_1";
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index d3b285cd5..da1662e51 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -399,6 +399,7 @@ void putLocal(String fullFilePath, byte[] data) {
       String stageLocation = this.fileTransferMetadataWithAge.localLocation;
       File destFile = Paths.get(stageLocation, fullFilePath).toFile();
       FileUtils.copyInputStreamToFile(input, destFile);
+      System.out.println("Filename: " + destFile); // TODO @rcheng - remove this before merge
     } catch (Exception ex) {
       throw new SFException(ex, ErrorCode.BLOB_UPLOAD_FAILURE);
     }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index cbc27ced0..a53cef4e7 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -8,6 +8,9 @@
 import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
 import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
 
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
@@ -37,6 +40,7 @@
 import javax.crypto.NoSuchPaddingException;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.Cryptor;
 import net.snowflake.ingest.utils.ErrorCode;
@@ -515,6 +519,9 @@ public void testBlobCreation() throws Exception {
 
   @Test
   public void testBuildAndUpload() throws Exception {
+    long expectedBuildLatencyMs = 100;
+    long expectedUploadLatencyMs = 200;
+
     TestContext<?> testContext = testContextFactory.create();
     SnowflakeStreamingIngestChannelInternal<?> channel1 = addChannel1(testContext);
     SnowflakeStreamingIngestChannelInternal<?> channel2 = addChannel2(testContext);
@@ -563,6 +570,15 @@ public void testBuildAndUpload() throws Exception {
     channel2Data.setRowSequencer(10L);
     channel2Data.setBufferSize(100);
 
+    // set client timers
+
+    SnowflakeStreamingIngestClientInternal client = testContext.client;
+    client.buildLatency = this.setupTimer(expectedBuildLatencyMs);
+    client.uploadLatency = this.setupTimer(expectedUploadLatencyMs);
+    client.uploadThroughput = Mockito.mock(Meter.class);
+    client.blobSizeHistogram = Mockito.mock(Histogram.class);
+    client.blobRowCountHistogram = Mockito.mock(Histogram.class);
+
     BlobMetadata blobMetadata = testContext.buildAndUpload();
 
     EpInfo expectedChunkEpInfo =
@@ -600,17 +616,15 @@ public void testBuildAndUpload() throws Exception {
     final ArgumentCaptor<List<ChunkMetadata>> metadataCaptor = ArgumentCaptor.forClass(List.class);
 
     Mockito.verify(testContext.flushService)
-        .upload(
-            nameCaptor.capture(),
-            blobCaptor.capture(),
-            metadataCaptor.capture(),
-            ArgumentMatchers.any());
+        .upload(nameCaptor.capture(), blobCaptor.capture(), metadataCaptor.capture(), ArgumentMatchers.anyLong());
     Assert.assertEquals("file_name", nameCaptor.getValue());
 
     ChunkMetadata metadataResult = metadataCaptor.getValue().get(0);
     List<ChannelMetadata> channelMetadataResult = metadataResult.getChannels();
 
     Assert.assertEquals(BlobBuilder.computeMD5(blobCaptor.getValue()), blobMetadata.getMD5());
+    Assert.assertEquals(blobMetadata.getBuildLatencyMs(), expectedBuildLatencyMs);
+    Assert.assertEquals(blobMetadata.getUploadLatencyMs(), expectedUploadLatencyMs);
 
     Assert.assertEquals(
         expectedChunkEpInfo.getRowCount(), metadataResult.getEpInfo().getRowCount());
@@ -875,4 +889,13 @@ public void testEncryptionDecryption()
 
     Assert.assertArrayEquals(data, decryptedData);
   }
+
+  private Timer setupTimer(long expectedLatencyMs) {
+    Timer.Context timerContext = Mockito.mock(Timer.Context.class);
+    Mockito.when(timerContext.stop()).thenReturn(expectedLatencyMs);
+    Timer timer = Mockito.mock(Timer.class);
+    Mockito.when(timer.time()).thenReturn(timerContext);
+
+    return timer;
+  }
 }

From 9134c43d826e479945d3d63337a9a0f2d5ab27fc Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Wed, 5 Apr 2023 16:44:18 -0700
Subject: [PATCH 165/356] verify the request has timestamps

---
 .../SnowflakeStreamingIngestExample.java      |  2 +-
 ...nowflakeStreamingIngestClientInternal.java |  7 +++++
 .../internal/StreamingIngestUtils.java        | 16 +++++-----
 .../streaming/internal/StreamingIngestIT.java | 29 ++++++++++++++-----
 4 files changed, 39 insertions(+), 15 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
index 0a565c97a..d52655150 100644
--- a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
+++ b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
@@ -28,7 +28,7 @@ public class SnowflakeStreamingIngestExample {
   // Please follow the example in profile_streaming.json.example to see the required properties, or
   // if you have already set up profile.json with Snowpipe before, all you need is to add the "role"
   // property.
-  private static String PROFILE_PATH = "/home/rcheng/tmp/profile.json.example";
+  private static String PROFILE_PATH = "profile.json";
   private static final ObjectMapper mapper = new ObjectMapper();
   private static final String TABLE_NAME = "revi_ingest_1";
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 7526b2117..dbbcb6562 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -75,6 +75,7 @@
 import net.snowflake.ingest.utils.Utils;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.util.VisibleForTesting;
 
 /**
  * The first version of implementation for SnowflakeStreamingIngestClient. The client internally
@@ -228,6 +229,12 @@ public SnowflakeStreamingIngestClientInternal(
     this(name, null, null, null, true, null, new HashMap<>());
   }
 
+  // TESTING ONLY - inject the request builder
+  @VisibleForTesting
+  public void injectRequestBuilder(RequestBuilder requestBuilder) {
+    this.requestBuilder = requestBuilder;
+  }
+
   /**
    * Get the client name
    *
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
index 88b406a62..e029d0c80 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
@@ -51,14 +51,8 @@ static <T extends StreamingIngestResponse> T executeWithRetries(
       CloseableHttpClient httpClient,
       RequestBuilder requestBuilder)
       throws IOException, IngestResponseException {
-    String payloadInString;
-    try {
-      payloadInString = objectMapper.writeValueAsString(payload);
-    } catch (JsonProcessingException e) {
-      throw new SFException(e, ErrorCode.BUILD_REQUEST_FAILURE, message);
-    }
     return executeWithRetries(
-        targetClass, endpoint, payloadInString, message, apiName, httpClient, requestBuilder);
+        targetClass, endpoint, getPayloadAsStr(payload, message), message, apiName, httpClient, requestBuilder);
   }
 
   static <T extends StreamingIngestResponse> T executeWithRetries(
@@ -122,4 +116,12 @@ public static String getShortname(final String fullname) {
     final String[] parts = fullname.split("/");
     return parts[parts.length - 1];
   }
+
+  public static String getPayloadAsStr(Map<Object, Object> payload, String message) {
+    try {
+      return objectMapper.writeValueAsString(payload);
+    } catch (JsonProcessingException e) {
+      throw new SFException(e, ErrorCode.BUILD_REQUEST_FAILURE, message);
+    }
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 83a1a1e9e..47336b39b 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -1,8 +1,6 @@
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
-import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
-import static net.snowflake.ingest.utils.Constants.ROLE;
+import static net.snowflake.ingest.utils.Constants.*;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.Is.is;
 
@@ -29,16 +27,16 @@
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 import java.util.function.IntConsumer;
+import java.util.regex.Pattern;
 import java.util.stream.IntStream;
+
 import net.snowflake.ingest.TestUtils;
+import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.ParameterProvider;
-import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.*;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -46,6 +44,9 @@
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
+import org.mockito.AdditionalMatchers;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
 
 /** Example streaming ingest sdk integration test */
 @RunWith(Parameterized.class)
@@ -117,6 +118,11 @@ public void afterAll() throws Exception {
 
   @Test
   public void testSimpleIngest() throws Exception {
+    // TODO @rcheng - dont want to change factory, so inject
+    RequestBuilder requestBuilder =
+            Mockito.spy(new RequestBuilder(new SnowflakeURL(TestUtils.getAccountURL()), TestUtils.getUser(), TestUtils.getKeyPair(), HttpUtil.getHttpClient(), "testrequestbuilder"));
+    client.injectRequestBuilder(requestBuilder);
+
     OpenChannelRequest request1 =
         OpenChannelRequest.builder("CHANNEL")
             .setDBName(testDb)
@@ -136,6 +142,15 @@ public void testSimpleIngest() throws Exception {
     // Close the channel after insertion
     channel1.close().get();
 
+    // verify expected request sent to server
+    String[] expectedPayloadParams = { "request_id", "blobs", "role", "flush_start_timestamp", "register_start_timestamp" };
+    for (String expectedParam : expectedPayloadParams) {
+      Mockito.verify(requestBuilder).generateStreamingIngestPostRequest(
+              ArgumentMatchers.contains(expectedParam),
+              ArgumentMatchers.refEq(REGISTER_BLOB_ENDPOINT),
+              ArgumentMatchers.refEq("register blob"));
+    }
+
     for (int i = 1; i < 15; i++) {
       if (channel1.getLatestCommittedOffsetToken() != null
           && channel1.getLatestCommittedOffsetToken().equals("999")) {

From 144b12eca4fc76c294624c51f639ca8c2ef12b38 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Wed, 5 Apr 2023 17:13:23 -0700
Subject: [PATCH 166/356] update tests

---
 .../streaming/internal/FlushService.java      |  3 +-
 .../streaming/internal/FlushServiceTest.java  | 14 +++++++--
 .../SnowflakeStreamingIngestClientTest.java   | 31 ++++++++++---------
 .../streaming/internal/StreamingIngestIT.java |  7 +++--
 4 files changed, 34 insertions(+), 21 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 64f543d52..297159cda 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -134,12 +134,13 @@ List<List<ChannelData<T>>> getData() {
       SnowflakeStreamingIngestClientInternal<T> client,
       ChannelCache<T> cache,
       StreamingIngestStage targetStage, // For testing
+      RegisterService registerService,
       boolean isTestMode) {
     this.owningClient = client;
     this.channelCache = cache;
     this.targetStage = targetStage;
     this.counter = new AtomicLong(0);
-    this.registerService = new RegisterService<>(client, isTestMode);
+    this.registerService = registerService;
     this.isNeedFlush = false;
     this.lastFlushTime = System.currentTimeMillis();
     this.isTestMode = isTestMode;
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index a53cef4e7..1f5cd8500 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -54,6 +54,7 @@
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
+import org.mockito.AdditionalMatchers;
 import org.mockito.ArgumentCaptor;
 import org.mockito.ArgumentMatchers;
 import org.mockito.Mockito;
@@ -92,6 +93,7 @@ private abstract static class TestContext<T> implements AutoCloseable {
     FlushService<T> flushService;
     StreamingIngestStage stage;
     ParameterProvider parameterProvider;
+    RegisterService registerService;
 
     final List<ChannelData<T>> channelData = new ArrayList<>();
 
@@ -103,7 +105,8 @@ private abstract static class TestContext<T> implements AutoCloseable {
       Mockito.when(client.getParameterProvider()).thenReturn(parameterProvider);
       channelCache = new ChannelCache<>();
       Mockito.when(client.getChannelCache()).thenReturn(channelCache);
-      flushService = Mockito.spy(new FlushService<>(client, channelCache, stage, false));
+      registerService = Mockito.spy(new RegisterService(client, client.isTestMode()));
+      flushService = Mockito.spy(new FlushService<>(client, channelCache, stage, registerService, false));
     }
 
     ChannelData<T> flushChannel(String name) {
@@ -460,28 +463,35 @@ public void testGetFilePath() {
 
   @Test
   public void testFlush() throws Exception {
+    long beginTestTimestamp = System.currentTimeMillis();
+
     TestContext<?> testContext = testContextFactory.create();
     FlushService<?> flushService = testContext.flushService;
+    RegisterService<?> registerService = testContext.registerService;
 
     // Nothing to flush
     flushService.flush(false).get();
     Mockito.verify(flushService, Mockito.times(0)).distributeFlushTasks();
+    Mockito.verify(registerService, Mockito.times(0)).registerBlobs(ArgumentMatchers.anyMap(), ArgumentMatchers.anyLong());
 
     // Force = true flushes
     flushService.flush(true).get();
     Mockito.verify(flushService).distributeFlushTasks();
     Mockito.verify(flushService, Mockito.times(1)).distributeFlushTasks();
+    Mockito.verify(registerService, Mockito.times(1)).registerBlobs(ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l > beginTestTimestamp));
 
     // isNeedFlush = true flushes
     flushService.isNeedFlush = true;
     flushService.flush(false).get();
     Mockito.verify(flushService, Mockito.times(2)).distributeFlushTasks();
+    Mockito.verify(registerService, Mockito.times(2)).registerBlobs(ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l > beginTestTimestamp));
     Assert.assertFalse(flushService.isNeedFlush);
 
     // lastFlushTime causes flush
     flushService.lastFlushTime = 0;
     flushService.flush(false).get();
     Mockito.verify(flushService, Mockito.times(3)).distributeFlushTasks();
+    Mockito.verify(registerService, Mockito.times(3)).registerBlobs(ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l > beginTestTimestamp));
     Assert.assertTrue(flushService.lastFlushTime > 0);
   }
 
@@ -755,7 +765,7 @@ public void testInvalidateChannels() {
     StreamingIngestStage stage = Mockito.mock(StreamingIngestStage.class);
     Mockito.when(stage.getClientPrefix()).thenReturn("client_prefix");
     FlushService<StubChunkData> flushService =
-        new FlushService<>(client, channelCache, stage, false);
+        new FlushService<>(client, channelCache, stage, new RegisterService(client, client.isTestMode()), false);
     flushService.invalidateAllChannelsInBlob(blobData);
 
     Assert.assertFalse(channel1.isValid());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index e799e1b7c..2f0186654 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -68,6 +68,7 @@
 
 public class SnowflakeStreamingIngestClientTest {
   private static final ObjectMapper objectMapper = new ObjectMapper();
+  private static final long flushTimestamp = System.currentTimeMillis();
 
   SnowflakeStreamingIngestChannelInternal<StubChunkData> channel1;
   SnowflakeStreamingIngestChannelInternal<StubChunkData> channel2;
@@ -487,7 +488,7 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
 
     List<BlobMetadata> blobs =
         Collections.singletonList(
-            new BlobMetadata("path", "md5", Collections.singletonList(chunkMetadata), null));
+            new BlobMetadata("path", "md5", Collections.singletonList(chunkMetadata)));
 
     Map<Object, Object> payload = new HashMap<>();
     payload.put("request_id", null);
@@ -587,8 +588,8 @@ private Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> getRetryBlobMetadata(
     chunks1.add(chunkMetadata1);
     chunks1.add(chunkMetadata2);
     chunks2.add(chunkMetadata3);
-    blobs.add(new BlobMetadata("path1", "md51", chunks1, null));
-    blobs.add(new BlobMetadata("path2", "md52", chunks2, null));
+    blobs.add(new BlobMetadata("path1", "md51", chunks1));
+    blobs.add(new BlobMetadata("path2", "md52", chunks2));
 
     List<ChannelRegisterStatus> channelRegisterStatuses = new ArrayList<>();
     ChannelRegisterStatus status1 = new ChannelRegisterStatus();
@@ -673,8 +674,8 @@ public void testRegisterBlobErrorResponse() throws Exception {
 
     try {
       List<BlobMetadata> blobs =
-          Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>(), null));
-      client.registerBlobs(blobs);
+          Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
+      client.registerBlobs(blobs, flushTimestamp);
       Assert.fail("Register blob should fail on 404 error");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.REGISTER_BLOB_FAILURE.getMessageCode(), e.getVendorCode());
@@ -720,8 +721,8 @@ public void testRegisterBlobSnowflakeInternalErrorResponse() throws Exception {
 
     try {
       List<BlobMetadata> blobs =
-          Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>(), null));
-      client.registerBlobs(blobs);
+          Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
+      client.registerBlobs(blobs, flushTimestamp);
       Assert.fail("Register blob should fail on SF internal error");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.REGISTER_BLOB_FAILURE.getMessageCode(), e.getVendorCode());
@@ -775,8 +776,8 @@ public void testRegisterBlobSuccessResponse() throws Exception {
             null);
 
     List<BlobMetadata> blobs =
-        Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>(), null));
-    client.registerBlobs(blobs);
+        Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
+    client.registerBlobs(blobs, flushTimestamp);
   }
 
   @Test
@@ -862,9 +863,9 @@ public void testRegisterBlobsRetries() throws Exception {
     client.getChannelCache().addChannel(channel2);
     client.getChannelCache().addChannel(channel3);
     client.getChannelCache().addChannel(channel4);
-    client.registerBlobs(blobs);
+    client.registerBlobs(blobs, flushTimestamp);
     Mockito.verify(requestBuilder, Mockito.times(MAX_STREAMING_INGEST_API_CHANNEL_RETRY + 1))
-        .generateStreamingIngestPostRequest(Mockito.anyString(), Mockito.any(), Mockito.any());
+        .generateStreamingIngestPostRequest(Mockito.contains(flushTimestamp + ""), Mockito.any(), Mockito.any());
     Assert.assertFalse(channel1.isValid());
     Assert.assertFalse(channel2.isValid());
   }
@@ -978,9 +979,9 @@ public void testRegisterBlobsRetriesSucceeds() throws Exception {
     client.getChannelCache().addChannel(channel3);
     client.getChannelCache().addChannel(channel4);
 
-    client.registerBlobs(blobs);
+    client.registerBlobs(blobs, flushTimestamp);
     Mockito.verify(requestBuilder, Mockito.times(2))
-        .generateStreamingIngestPostRequest(Mockito.anyString(), Mockito.any(), Mockito.any());
+        .generateStreamingIngestPostRequest(Mockito.contains(flushTimestamp + ""), Mockito.any(), Mockito.any());
     Assert.assertTrue(channel1.isValid());
     Assert.assertTrue(channel2.isValid());
   }
@@ -1082,8 +1083,8 @@ public void testRegisterBlobResponseWithInvalidChannel() throws Exception {
     Assert.assertTrue(channel2.isValid());
 
     List<BlobMetadata> blobs =
-        Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>(), null));
-    client.registerBlobs(blobs);
+        Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
+    client.registerBlobs(blobs, flushTimestamp);
 
     // Channel2 should be invalidated now
     Assert.assertTrue(channel1.isValid());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 47336b39b..18b139439 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -27,7 +27,6 @@
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 import java.util.function.IntConsumer;
-import java.util.regex.Pattern;
 import java.util.stream.IntStream;
 
 import net.snowflake.ingest.TestUtils;
@@ -36,7 +35,10 @@
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
-import net.snowflake.ingest.utils.*;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.ParameterProvider;
+import net.snowflake.ingest.utils.SFException;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -44,7 +46,6 @@
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
-import org.mockito.AdditionalMatchers;
 import org.mockito.ArgumentMatchers;
 import org.mockito.Mockito;
 

From 2f65cdd9c72fb80915e016c1a773903425aeb278 Mon Sep 17 00:00:00 2001
From: Tyler Jones <59716821+sfc-gh-tjones@users.noreply.github.com>
Date: Mon, 3 Apr 2023 07:49:26 -0700
Subject: [PATCH 167/356] Build fixes (#428)

This diff does the following:

- Removes all linker checker issues
- Fixes a warning about brackets in `ParameterProvider.java`
---
 linkage-checker-exclusion-rules.xml           | 185 +++++++++++++++++-
 .../ingest/utils/ParameterProvider.java       |   2 +-
 2 files changed, 181 insertions(+), 6 deletions(-)

diff --git a/linkage-checker-exclusion-rules.xml b/linkage-checker-exclusion-rules.xml
index 0139a7d76..c26b7dbc4 100644
--- a/linkage-checker-exclusion-rules.xml
+++ b/linkage-checker-exclusion-rules.xml
@@ -23,12 +23,16 @@
     <Source><Package name="org.apache.hadoop.mapred"/></Source>
     <Reason>Conflict due to parquet-hadoop uses provided scope of hadoo-common</Reason>
   </LinkageError>
+  <LinkageError>
+    <Target><Package name="net.snowflake.ingest.internal.apache.hadoop.mapreduce"/></Target>
+    <Reason>There is no way we use mapreduce in this SDK</Reason>
+  </LinkageError>
   <LinkageError>
     <Source><Package name="net.snowflake.client.jdbc.internal"/></Source>
     <Reason>Unknown: Likely the referenced class is not used in the jdbc package</Reason>
   </LinkageError>
   <LinkageError>
-    <Source><Class name="net.snowflake.ingest.internal.com.nimbusds.jose.crypto.impl.LegacyAESGCM"/></Source>
+    <Source><Package name="net.snowflake.ingest.internal.com.nimbusds.jose.crypto"/></Source>
     <Reason>Not Used, appears to be Legacy</Reason>
   </LinkageError>
   <LinkageError>
@@ -36,12 +40,183 @@
     <Reason>Not Used</Reason>
   </LinkageError>
   <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.com.nimbusds.jose.crypto.bc"/></Source>
+    <Source><Class name="net.snowflake.ingest.internal.com.nimbusds.jose.util.X509CertChainUtils"/></Source>
     <Reason>Not Used</Reason>
   </LinkageError>
   <LinkageError>
-    <Source><Class name="net.snowflake.ingest.internal.com.nimbusds.jose.util.X509CertChainUtils"/></Source>
+    <Source><Package name="net.snowflake.ingest.internal.apache.commons.configuration2"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.metrics2"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.fs"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.fs"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Class name="net.snowflake.ingest.internal.apache.hadoop.crypto.key.KeyShell "/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.security"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.ipc"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.net"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Class name="net.snowflake.ingest.internal.apache.conf.StorageSize"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Class name="net.snowflake.ingest.internal.apache.commons.text.lookup.DateStringLookup"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.apache.commons.lang3"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.classification"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.apache.yetus"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="net.snowflake.ingest.internal.apache.commons.lang3.time.FastDateFormat"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.io.nativeio"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.apache.commons.text"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="net.snowflake.ingest.internal.apache.commons.lang3.SystemUtils"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="net.snowflake.ingest.internal.apache.commons.lang3.StringUtils"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="net.snowflake.ingest.internal.apache.commons.lang3.builder.EqualsBuilder"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="net.snowflake.ingest.internal.apache.commons.lang3.builder.HashCodeBuilder"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Class name="javax.activation.MimetypesFileTypeMap"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Package name="org.eclipse.jetty"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Package name="net.snowflake.ingest.internal.fasterxml.jackson.jaxrs"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Class name="com.sun.jersey.json.impl.MoxyXmlStructure"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="com.sun.xml.bind.v2.runtime.output"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Package name="com.sun.xml.bind.v2.runtime.unmarshaller"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Class name="net.snowflake.ingest.internal.apache.commons.logging.impl.LogKitLogger"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="org.joda.time.DateMidnight"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="org.joda.time.DateMidnight"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Class name="org.codehaus.jackson.map.ext.JodaSerializers"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Class name="org.codehaus.jackson.map.ext.JodaDeserializers"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="net.snowflake.ingest.internal.com.nimbusds.jwt.ReadOnlyJWTClaimsSet"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Class name="jcraft.jsch.jcraft.Compression"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="org.brotli.dec.BrotliInputStream"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Class name="com.ctc.wstx.msv.GenericMsvValidator"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Package name="net.snowflake.ingest.internal.apache.hadoop.mapred"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Package name="com.sun.msv.verifier.regexp.xmlschema"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="com.sun.javadoc.DocErrorReporter"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="com.sun.msv.reader.xmlschema.XMLSchemaReader"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="com.sun.msv.verifier.regexp.xmlschema.XSREDocDecl"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="com.sun.msv.reader.util.IgnoreController"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="com.jcraft.jzlib.ZStream"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="com.sun.msv.verifier.regexp.REDocumentDeclaration"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="com.sun.msv.reader.trex.ng.RELAXNGReader"/></Target>
     <Reason>Not Used</Reason>
   </LinkageError>
-</LinkageCheckerFilter>
-
+</LinkageCheckerFilter>
\ No newline at end of file
diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
index 07bca85cc..93077bbf8 100644
--- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
+++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -47,7 +47,7 @@ public class ParameterProvider {
    * Constructor. Takes properties from profile file and properties from client constructor and
    * resolves final parameter value
    *
-   * @param parameterOverrides Map<String, Object> of parameter name -> value
+   * @param parameterOverrides Map of parameter name to value
    * @param props Properties from profile file
    */
   public ParameterProvider(Map<String, Object> parameterOverrides, Properties props) {

From 9f0418786f65d39d0997c2843c49602c53cf17ae Mon Sep 17 00:00:00 2001
From: Tyler Jones <59716821+sfc-gh-tjones@users.noreply.github.com>
Date: Tue, 4 Apr 2023 14:45:32 -0700
Subject: [PATCH 168/356] Fixes snyk issue identified in issue 422 (#430)

* Fixes snyk issue identified in issue 422
* Removes mapreduce client
* Removes Zookeeper and hadoop-auth
* Accounts for error code that changed
---
 linkage-checker-exclusion-rules.xml           | 58 ++++++++++++++++++-
 pom.xml                                       | 14 +++++
 .../internal/datatypes/StringsIT.java         |  2 +-
 3 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/linkage-checker-exclusion-rules.xml b/linkage-checker-exclusion-rules.xml
index c26b7dbc4..946b0b480 100644
--- a/linkage-checker-exclusion-rules.xml
+++ b/linkage-checker-exclusion-rules.xml
@@ -23,10 +23,22 @@
     <Source><Package name="org.apache.hadoop.mapred"/></Source>
     <Reason>Conflict due to parquet-hadoop uses provided scope of hadoo-common</Reason>
   </LinkageError>
+  <LinkageError>
+    <Target><Package name="org.apache.hadoop.security.authentication"/></Target>
+    <Reason>We do not need hadoop authentication</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Class name="org.apache.hadoop.util.PlatformName"/></Target>
+    <Reason>Relates to security/SSL pulled in by hadoop-auth, not used</Reason>
+  </LinkageError>
   <LinkageError>
     <Target><Package name="net.snowflake.ingest.internal.apache.hadoop.mapreduce"/></Target>
     <Reason>There is no way we use mapreduce in this SDK</Reason>
   </LinkageError>
+  <LinkageError>
+    <Target><Package name="net.snowflake.ingest.internal.apache.hadoop.zookeeper"/></Target>
+    <Reason>There is no way we use Zookeeper in this SDK</Reason>
+  </LinkageError>
   <LinkageError>
     <Source><Package name="net.snowflake.client.jdbc.internal"/></Source>
     <Reason>Unknown: Likely the referenced class is not used in the jdbc package</Reason>
@@ -131,6 +143,14 @@
     <Target><Package name="org.eclipse.jetty"/></Target>
     <Reason>Not Used</Reason>
   </LinkageError>
+  <LinkageError>
+    <Target><Package name="net.snowflake.ingest.internal.apache.zookeeper"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Package name="org.apache.zookeeper"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
   <LinkageError>
     <Target><Package name="net.snowflake.ingest.internal.fasterxml.jackson.jaxrs"/></Target>
     <Reason>Not Used</Reason>
@@ -183,10 +203,22 @@
     <Source><Class name="com.ctc.wstx.msv.GenericMsvValidator"/></Source>
     <Reason>Not Used</Reason>
   </LinkageError>
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.apache.commons.compress.harmony"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
   <LinkageError>
     <Target><Package name="net.snowflake.ingest.internal.apache.hadoop.mapred"/></Target>
     <Reason>Not Used</Reason>
   </LinkageError>
+  <LinkageError>
+    <Target><Package name="net.snowflake.ingest.internal.apache.hadoop.security"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Package name="net.snowflake.ingest.internal.net.objectweb"/></Target>
+    <Reason>Not Used</Reason>
+  </LinkageError>
   <LinkageError>
     <Target><Package name="com.sun.msv.verifier.regexp.xmlschema"/></Target>
     <Reason>Not Used</Reason>
@@ -219,4 +251,28 @@
     <Target><Class name="com.sun.msv.reader.trex.ng.RELAXNGReader"/></Target>
     <Reason>Not Used</Reason>
   </LinkageError>
-</LinkageCheckerFilter>
\ No newline at end of file
+  <LinkageError>
+    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.log"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Class name="org.apache.hadoop.log.LogLevel"/></Source>
+    <Reason>Not Used</Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Class name="org.apache.hadoop.log.metrics.EventCounter"/></Source>
+    <Reason></Reason>
+  </LinkageError>
+  <LinkageError>
+    <Source><Class name="org.apache.hadoop.log.metrics.Log4Json"/></Source>
+    <Reason></Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Package name="org.apache.log4j"/></Target>
+    <Reason></Reason>
+  </LinkageError>
+  <LinkageError>
+    <Target><Package name="net.snowflake.ingest.internal.apache.log4j"/></Target>
+    <Reason></Reason>
+  </LinkageError>
+</LinkageCheckerFilter>
diff --git a/pom.xml b/pom.xml
index 8574481ed..d2baff4f6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -51,6 +51,7 @@
     <jacoco.version>0.8.5</jacoco.version>
     <maven.compiler.source>1.8</maven.compiler.source>
     <maven.compiler.target>1.8</maven.compiler.target>
+    <net.minidev.version>2.4.9</net.minidev.version>
     <netty.version>4.1.82.Final</netty.version>
     <nimbusds.version>9.9.3</nimbusds.version>
     <objenesis.version>3.1</objenesis.version>
@@ -112,6 +113,11 @@
         <artifactId>netty-common</artifactId>
         <version>${netty.version}</version>
       </dependency>
+      <dependency>
+        <groupId>net.minidev</groupId>
+        <artifactId>json-smart</artifactId>
+        <version>${net.minidev.version}</version>
+      </dependency>
       <dependency>
         <groupId>net.snowflake</groupId>
         <artifactId>snowflake-jdbc</artifactId>
@@ -145,6 +151,14 @@
             <groupId>javax.activation</groupId>
             <artifactId>activation</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-auth</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.zookeeper</groupId>
+            <artifactId>zookeeper</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-log4j12</artifactId>
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
index f41e24dc1..cfa59e0b1 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
@@ -217,7 +217,7 @@ public void testCollatedColumnsNotSupported() throws SQLException {
       openChannel(tableName);
       Assert.fail("Opening a channel shouldn't have succeeded");
     } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.UNSUPPORTED_DATA_TYPE.getMessageCode(), e.getVendorCode());
+      Assert.assertEquals(ErrorCode.OPEN_CHANNEL_FAILURE.getMessageCode(), e.getVendorCode());
     }
   }
 }

From 9e1f5e9c8a18769ef1eb0bc0fea0c95262780867 Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Wed, 5 Apr 2023 13:19:03 +0200
Subject: [PATCH 169/356] SNOW-773778 make the absolute size threshold limit
 for throttling configurable, change the default from 100MB to 200 MB (#426)

---
 ...owflakeStreamingIngestChannelInternal.java |  5 +++--
 .../net/snowflake/ingest/utils/Constants.java |  1 -
 .../ingest/utils/ParameterProvider.java       | 21 +++++++++++++++++++
 .../internal/ParameterProviderTest.java       | 12 +++++++++++
 .../streaming/internal/StreamingIngestIT.java |  1 +
 5 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 9a935b0a7..102ee5535 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -5,7 +5,6 @@
 package net.snowflake.ingest.streaming.internal;
 
 import static net.snowflake.ingest.utils.Constants.INSERT_THROTTLE_MAX_RETRY_COUNT;
-import static net.snowflake.ingest.utils.Constants.LOW_RUNTIME_MEMORY_THRESHOLD_IN_BYTES;
 import static net.snowflake.ingest.utils.Constants.MAX_CHUNK_SIZE_IN_BYTES;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
 import static net.snowflake.ingest.utils.ParameterProvider.MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT;
@@ -428,6 +427,8 @@ void throttleInsertIfNeeded(Runtime runtime) {
 
   /** Check whether we have a low runtime memory condition */
   private boolean hasLowRuntimeMemory(Runtime runtime) {
+    int insertThrottleThresholdInBytes =
+        this.owningClient.getParameterProvider().getInsertThrottleThresholdInBytes();
     int insertThrottleThresholdInPercentage =
         this.owningClient.getParameterProvider().getInsertThrottleThresholdInPercentage();
     long maxMemoryLimitInBytes =
@@ -438,7 +439,7 @@ private boolean hasLowRuntimeMemory(Runtime runtime) {
             : maxMemoryLimitInBytes;
     long freeMemory = runtime.freeMemory() + (runtime.maxMemory() - runtime.totalMemory());
     boolean hasLowRuntimeMemory =
-        freeMemory < LOW_RUNTIME_MEMORY_THRESHOLD_IN_BYTES
+        freeMemory < insertThrottleThresholdInBytes
             && freeMemory * 100 / maxMemory < insertThrottleThresholdInPercentage;
     if (hasLowRuntimeMemory) {
       logger.logWarn(
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index 13c3c5c3e..e43c419c8 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -51,7 +51,6 @@ public class Constants {
   public static final int ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES = 16;
   public static final int MAX_STREAMING_INGEST_API_CHANNEL_RETRY = 3;
   public static final int STREAMING_INGEST_TELEMETRY_UPLOAD_INTERVAL_IN_SEC = 10;
-  public static final int LOW_RUNTIME_MEMORY_THRESHOLD_IN_BYTES = 100 * 1024 * 1024;
   public static final long EP_NDV_UNKNOWN = -1L;
 
   // Channel level constants
diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
index 93077bbf8..11f9fbeac 100644
--- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
+++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -14,6 +14,9 @@ public class ParameterProvider {
       "STREAMING_INGEST_CLIENT_SDK_INSERT_THROTTLE_INTERVAL_IN_MILLIS".toLowerCase();
   public static final String INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE =
       "STREAMING_INGEST_CLIENT_SDK_INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE".toLowerCase();
+  public static final String INSERT_THROTTLE_THRESHOLD_IN_BYTES =
+      "STREAMING_INGEST_CLIENT_SDK_INSERT_THROTTLE_THRESHOLD_IN_BYTES".toLowerCase();
+
   public static final String ENABLE_SNOWPIPE_STREAMING_METRICS =
       "ENABLE_SNOWPIPE_STREAMING_JMX_METRICS".toLowerCase();
   public static final String BLOB_FORMAT_VERSION = "BLOB_FORMAT_VERSION".toLowerCase();
@@ -29,6 +32,7 @@ public class ParameterProvider {
   public static final long BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_DEFAULT = 100;
   public static final long INSERT_THROTTLE_INTERVAL_IN_MILLIS_DEFAULT = 1000;
   public static final int INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_DEFAULT = 10;
+  public static final int INSERT_THROTTLE_THRESHOLD_IN_BYTES_DEFAULT = 200 * 1024 * 1024; // 200MB
   public static final boolean SNOWPIPE_STREAMING_METRICS_DEFAULT = false;
   public static final Constants.BdecVersion BLOB_FORMAT_VERSION_DEFAULT =
       Constants.BdecVersion.THREE;
@@ -101,6 +105,12 @@ private void setParameterMap(Map<String, Object> parameterOverrides, Properties
         parameterOverrides,
         props);
 
+    this.updateValue(
+        INSERT_THROTTLE_THRESHOLD_IN_BYTES,
+        INSERT_THROTTLE_THRESHOLD_IN_BYTES_DEFAULT,
+        parameterOverrides,
+        props);
+
     this.updateValue(
         ENABLE_SNOWPIPE_STREAMING_METRICS,
         SNOWPIPE_STREAMING_METRICS_DEFAULT,
@@ -173,6 +183,17 @@ public int getInsertThrottleThresholdInPercentage() {
     return (int) val;
   }
 
+  /** @return Absolute size in bytes of free total memory at which we throttle row inserts */
+  public int getInsertThrottleThresholdInBytes() {
+    Object val =
+        this.parameterMap.getOrDefault(
+            INSERT_THROTTLE_THRESHOLD_IN_BYTES, INSERT_THROTTLE_THRESHOLD_IN_BYTES_DEFAULT);
+    if (val instanceof String) {
+      return Integer.parseInt(val.toString());
+    }
+    return (int) val;
+  }
+
   /** @return true if jmx metrics are enabled for a client */
   public boolean hasEnabledSnowpipeStreamingMetrics() {
     Object val =
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
index b7b1a1c74..f7f8da84f 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
@@ -16,6 +16,7 @@ public void withValuesSet() {
     parameterMap.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS, 3L);
     parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 4L);
     parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE, 6);
+    parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_BYTES, 1024);
     parameterMap.put(ParameterProvider.INSERT_THROTTLE_INTERVAL_IN_MILLIS, 7L);
     parameterMap.put(ParameterProvider.IO_TIME_CPU_RATIO, 10);
     parameterMap.put(ParameterProvider.BLOB_UPLOAD_MAX_RETRY_COUNT, 100);
@@ -25,6 +26,7 @@ public void withValuesSet() {
     Assert.assertEquals(3L, parameterProvider.getBufferFlushIntervalInMs());
     Assert.assertEquals(4L, parameterProvider.getBufferFlushCheckIntervalInMs());
     Assert.assertEquals(6, parameterProvider.getInsertThrottleThresholdInPercentage());
+    Assert.assertEquals(1024, parameterProvider.getInsertThrottleThresholdInBytes());
     Assert.assertEquals(7L, parameterProvider.getInsertThrottleIntervalInMs());
     Assert.assertEquals(10, parameterProvider.getIOTimeCpuRatio());
     Assert.assertEquals(100, parameterProvider.getBlobUploadMaxRetryCount());
@@ -37,11 +39,13 @@ public void withNullProps() {
     parameterMap.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS, 3L);
     parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 4L);
     parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE, 6);
+    parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_BYTES, 1024);
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, null);
 
     Assert.assertEquals(3, parameterProvider.getBufferFlushIntervalInMs());
     Assert.assertEquals(4, parameterProvider.getBufferFlushCheckIntervalInMs());
     Assert.assertEquals(6, parameterProvider.getInsertThrottleThresholdInPercentage());
+    Assert.assertEquals(1024, parameterProvider.getInsertThrottleThresholdInBytes());
     Assert.assertEquals(
         ParameterProvider.INSERT_THROTTLE_INTERVAL_IN_MILLIS_DEFAULT,
         parameterProvider.getInsertThrottleIntervalInMs());
@@ -53,11 +57,13 @@ public void withNullParameterMap() {
     props.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS, 3L);
     props.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 4L);
     props.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE, 6);
+    props.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_BYTES, 1024);
     ParameterProvider parameterProvider = new ParameterProvider(null, props);
 
     Assert.assertEquals(3, parameterProvider.getBufferFlushIntervalInMs());
     Assert.assertEquals(4, parameterProvider.getBufferFlushCheckIntervalInMs());
     Assert.assertEquals(6, parameterProvider.getInsertThrottleThresholdInPercentage());
+    Assert.assertEquals(1024, parameterProvider.getInsertThrottleThresholdInBytes());
     Assert.assertEquals(
         ParameterProvider.INSERT_THROTTLE_INTERVAL_IN_MILLIS_DEFAULT,
         parameterProvider.getInsertThrottleIntervalInMs());
@@ -76,6 +82,9 @@ public void withNullInputs() {
     Assert.assertEquals(
         ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_DEFAULT,
         parameterProvider.getInsertThrottleThresholdInPercentage());
+    Assert.assertEquals(
+        ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_BYTES_DEFAULT,
+        parameterProvider.getInsertThrottleThresholdInBytes());
     Assert.assertEquals(
         ParameterProvider.INSERT_THROTTLE_INTERVAL_IN_MILLIS_DEFAULT,
         parameterProvider.getInsertThrottleIntervalInMs());
@@ -94,6 +103,9 @@ public void withDefaultValues() {
     Assert.assertEquals(
         ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_DEFAULT,
         parameterProvider.getInsertThrottleThresholdInPercentage());
+    Assert.assertEquals(
+        ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_BYTES_DEFAULT,
+        parameterProvider.getInsertThrottleThresholdInBytes());
     Assert.assertEquals(
         ParameterProvider.INSERT_THROTTLE_INTERVAL_IN_MILLIS_DEFAULT,
         parameterProvider.getInsertThrottleIntervalInMs());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 18b139439..686fb9a36 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -202,6 +202,7 @@ public void testParameterOverrides() throws Exception {
     parameterMap.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS, 30L);
     parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 50L);
     parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE, 1);
+    parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_BYTES, 1024);
     parameterMap.put(ParameterProvider.INSERT_THROTTLE_INTERVAL_IN_MILLIS, 1L);
     parameterMap.put(ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS, true);
     parameterMap.put(ParameterProvider.IO_TIME_CPU_RATIO, 1);

From 0e22ed0871eb531cc9c0418e5a95c03fc776c371 Mon Sep 17 00:00:00 2001
From: Tyler Jones <59716821+sfc-gh-tjones@users.noreply.github.com>
Date: Wed, 5 Apr 2023 15:27:30 -0700
Subject: [PATCH 170/356] Issue 421 Fixes Jettison vuln (#434)

---
 pom.xml                                                      | 5 +++++
 .../ingest/streaming/internal/datatypes/StringsIT.java       | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index d2baff4f6..50acbf777 100644
--- a/pom.xml
+++ b/pom.xml
@@ -209,6 +209,11 @@
         <artifactId>jackson-xc</artifactId>
         <version>${codehaus.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.codehaus.jettison</groupId>
+        <artifactId>jettison</artifactId>
+        <version>1.5.2</version>
+      </dependency>
       <dependency>
         <groupId>org.codehaus.woodstox</groupId>
         <artifactId>stax2-api</artifactId>
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
index cfa59e0b1..f41e24dc1 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
@@ -217,7 +217,7 @@ public void testCollatedColumnsNotSupported() throws SQLException {
       openChannel(tableName);
       Assert.fail("Opening a channel shouldn't have succeeded");
     } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.OPEN_CHANNEL_FAILURE.getMessageCode(), e.getVendorCode());
+      Assert.assertEquals(ErrorCode.UNSUPPORTED_DATA_TYPE.getMessageCode(), e.getVendorCode());
     }
   }
 }

From 3e239c3c09473d9825176de0376f8b0439bdcce7 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Wed, 5 Apr 2023 17:28:52 -0700
Subject: [PATCH 171/356] format

---
 .../internal/StreamingIngestUtils.java        |  8 ++++-
 .../streaming/internal/FlushServiceTest.java  | 29 +++++++++++++------
 .../SnowflakeStreamingIngestClientTest.java   |  6 ++--
 .../streaming/internal/StreamingIngestIT.java | 18 +++++++++---
 4 files changed, 45 insertions(+), 16 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
index e029d0c80..5d5e10435 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
@@ -52,7 +52,13 @@ static <T extends StreamingIngestResponse> T executeWithRetries(
       RequestBuilder requestBuilder)
       throws IOException, IngestResponseException {
     return executeWithRetries(
-        targetClass, endpoint, getPayloadAsStr(payload, message), message, apiName, httpClient, requestBuilder);
+        targetClass,
+        endpoint,
+        getPayloadAsStr(payload, message),
+        message,
+        apiName,
+        httpClient,
+        requestBuilder);
   }
 
   static <T extends StreamingIngestResponse> T executeWithRetries(
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 1f5cd8500..fadda7979 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -40,7 +40,6 @@
 import javax.crypto.NoSuchPaddingException;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.Cryptor;
 import net.snowflake.ingest.utils.ErrorCode;
@@ -54,7 +53,6 @@
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
-import org.mockito.AdditionalMatchers;
 import org.mockito.ArgumentCaptor;
 import org.mockito.ArgumentMatchers;
 import org.mockito.Mockito;
@@ -106,7 +104,8 @@ private abstract static class TestContext<T> implements AutoCloseable {
       channelCache = new ChannelCache<>();
       Mockito.when(client.getChannelCache()).thenReturn(channelCache);
       registerService = Mockito.spy(new RegisterService(client, client.isTestMode()));
-      flushService = Mockito.spy(new FlushService<>(client, channelCache, stage, registerService, false));
+      flushService =
+          Mockito.spy(new FlushService<>(client, channelCache, stage, registerService, false));
     }
 
     ChannelData<T> flushChannel(String name) {
@@ -472,26 +471,33 @@ public void testFlush() throws Exception {
     // Nothing to flush
     flushService.flush(false).get();
     Mockito.verify(flushService, Mockito.times(0)).distributeFlushTasks();
-    Mockito.verify(registerService, Mockito.times(0)).registerBlobs(ArgumentMatchers.anyMap(), ArgumentMatchers.anyLong());
+    Mockito.verify(registerService, Mockito.times(0))
+        .registerBlobs(ArgumentMatchers.anyMap(), ArgumentMatchers.anyLong());
 
     // Force = true flushes
     flushService.flush(true).get();
     Mockito.verify(flushService).distributeFlushTasks();
     Mockito.verify(flushService, Mockito.times(1)).distributeFlushTasks();
-    Mockito.verify(registerService, Mockito.times(1)).registerBlobs(ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l > beginTestTimestamp));
+    Mockito.verify(registerService, Mockito.times(1))
+        .registerBlobs(
+            ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l > beginTestTimestamp));
 
     // isNeedFlush = true flushes
     flushService.isNeedFlush = true;
     flushService.flush(false).get();
     Mockito.verify(flushService, Mockito.times(2)).distributeFlushTasks();
-    Mockito.verify(registerService, Mockito.times(2)).registerBlobs(ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l > beginTestTimestamp));
+    Mockito.verify(registerService, Mockito.times(2))
+        .registerBlobs(
+            ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l > beginTestTimestamp));
     Assert.assertFalse(flushService.isNeedFlush);
 
     // lastFlushTime causes flush
     flushService.lastFlushTime = 0;
     flushService.flush(false).get();
     Mockito.verify(flushService, Mockito.times(3)).distributeFlushTasks();
-    Mockito.verify(registerService, Mockito.times(3)).registerBlobs(ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l > beginTestTimestamp));
+    Mockito.verify(registerService, Mockito.times(3))
+        .registerBlobs(
+            ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l > beginTestTimestamp));
     Assert.assertTrue(flushService.lastFlushTime > 0);
   }
 
@@ -626,7 +632,11 @@ public void testBuildAndUpload() throws Exception {
     final ArgumentCaptor<List<ChunkMetadata>> metadataCaptor = ArgumentCaptor.forClass(List.class);
 
     Mockito.verify(testContext.flushService)
-        .upload(nameCaptor.capture(), blobCaptor.capture(), metadataCaptor.capture(), ArgumentMatchers.anyLong());
+        .upload(
+            nameCaptor.capture(),
+            blobCaptor.capture(),
+            metadataCaptor.capture(),
+            ArgumentMatchers.anyLong());
     Assert.assertEquals("file_name", nameCaptor.getValue());
 
     ChunkMetadata metadataResult = metadataCaptor.getValue().get(0);
@@ -765,7 +775,8 @@ public void testInvalidateChannels() {
     StreamingIngestStage stage = Mockito.mock(StreamingIngestStage.class);
     Mockito.when(stage.getClientPrefix()).thenReturn("client_prefix");
     FlushService<StubChunkData> flushService =
-        new FlushService<>(client, channelCache, stage, new RegisterService(client, client.isTestMode()), false);
+        new FlushService<>(
+            client, channelCache, stage, new RegisterService(client, client.isTestMode()), false);
     flushService.invalidateAllChannelsInBlob(blobData);
 
     Assert.assertFalse(channel1.isValid());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index 2f0186654..4e162da69 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -865,7 +865,8 @@ public void testRegisterBlobsRetries() throws Exception {
     client.getChannelCache().addChannel(channel4);
     client.registerBlobs(blobs, flushTimestamp);
     Mockito.verify(requestBuilder, Mockito.times(MAX_STREAMING_INGEST_API_CHANNEL_RETRY + 1))
-        .generateStreamingIngestPostRequest(Mockito.contains(flushTimestamp + ""), Mockito.any(), Mockito.any());
+        .generateStreamingIngestPostRequest(
+            Mockito.contains(flushTimestamp + ""), Mockito.any(), Mockito.any());
     Assert.assertFalse(channel1.isValid());
     Assert.assertFalse(channel2.isValid());
   }
@@ -981,7 +982,8 @@ public void testRegisterBlobsRetriesSucceeds() throws Exception {
 
     client.registerBlobs(blobs, flushTimestamp);
     Mockito.verify(requestBuilder, Mockito.times(2))
-        .generateStreamingIngestPostRequest(Mockito.contains(flushTimestamp + ""), Mockito.any(), Mockito.any());
+        .generateStreamingIngestPostRequest(
+            Mockito.contains(flushTimestamp + ""), Mockito.any(), Mockito.any());
     Assert.assertTrue(channel1.isValid());
     Assert.assertTrue(channel2.isValid());
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 686fb9a36..e302e3610 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -28,7 +28,6 @@
 import java.util.function.Consumer;
 import java.util.function.IntConsumer;
 import java.util.stream.IntStream;
-
 import net.snowflake.ingest.TestUtils;
 import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
@@ -37,8 +36,10 @@
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.HttpUtil;
 import net.snowflake.ingest.utils.ParameterProvider;
 import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.SnowflakeURL;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -121,7 +122,13 @@ public void afterAll() throws Exception {
   public void testSimpleIngest() throws Exception {
     // TODO @rcheng - dont want to change factory, so inject
     RequestBuilder requestBuilder =
-            Mockito.spy(new RequestBuilder(new SnowflakeURL(TestUtils.getAccountURL()), TestUtils.getUser(), TestUtils.getKeyPair(), HttpUtil.getHttpClient(), "testrequestbuilder"));
+        Mockito.spy(
+            new RequestBuilder(
+                new SnowflakeURL(TestUtils.getAccountURL()),
+                TestUtils.getUser(),
+                TestUtils.getKeyPair(),
+                HttpUtil.getHttpClient(),
+                "testrequestbuilder"));
     client.injectRequestBuilder(requestBuilder);
 
     OpenChannelRequest request1 =
@@ -144,9 +151,12 @@ public void testSimpleIngest() throws Exception {
     channel1.close().get();
 
     // verify expected request sent to server
-    String[] expectedPayloadParams = { "request_id", "blobs", "role", "flush_start_timestamp", "register_start_timestamp" };
+    String[] expectedPayloadParams = {
+      "request_id", "blobs", "role", "flush_start_timestamp", "register_start_timestamp"
+    };
     for (String expectedParam : expectedPayloadParams) {
-      Mockito.verify(requestBuilder).generateStreamingIngestPostRequest(
+      Mockito.verify(requestBuilder)
+          .generateStreamingIngestPostRequest(
               ArgumentMatchers.contains(expectedParam),
               ArgumentMatchers.refEq(REGISTER_BLOB_ENDPOINT),
               ArgumentMatchers.refEq("register blob"));

From 4c22f24b468964914f376e99104b3ea07ed23ac9 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Wed, 5 Apr 2023 17:33:57 -0700
Subject: [PATCH 172/356] ms not ns

---
 .../streaming/internal/FlushService.java      | 24 ++++++++++++++-----
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 297159cda..fee2ef20f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -493,9 +493,14 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
     // Construct the blob along with the metadata of the blob
     BlobBuilder.Blob blob = BlobBuilder.constructBlobAndMetadata(filePath, blobData, bdecVersion);
 
-    blob.blobLatencies.setBuildLatencyMs(buildContext);
-
-    return upload(filePath, blob.blobBytes, blob.chunksMetadataList, blob.blobLatencies);
+    return buildContext != null
+        ? upload(
+            filePath,
+            blob.blobBytes,
+            blob.chunksMetadataList,
+            TimeUnit.NANOSECONDS.toMillis(buildContext.stop()))
+        : upload(
+            filePath, blob.blobBytes, blob.chunksMetadataList, BlobMetadata.DEFAULT_BLOB_LATENCY);
   }
 
   /**
@@ -504,15 +509,17 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
    * @param filePath full path of the blob file
    * @param blob blob data
    * @param metadata a list of chunk metadata
+   * @param buildLatencyMs the build latency for the blob
    * @return BlobMetadata object used to create the register blob request
    */
   BlobMetadata upload(
-      String filePath, byte[] blob, List<ChunkMetadata> metadata, BlobLatencies blobLatencies)
+      String filePath, byte[] blob, List<ChunkMetadata> metadata, long buildLatencyMs)
       throws NoSuchAlgorithmException {
     logger.logInfo("Start uploading file={}, size={}", filePath, blob.length);
     long startTime = System.currentTimeMillis();
 
     Timer.Context uploadContext = Utils.createTimerContext(this.owningClient.uploadLatency);
+    long uploadLatencyMs = BlobMetadata.DEFAULT_BLOB_LATENCY;
 
     this.targetStage.put(filePath, blob);
 
@@ -523,7 +530,7 @@ BlobMetadata upload(
         System.currentTimeMillis() - startTime);
 
     if (uploadContext != null) {
-      blobLatencies.setUploadLatencyMs(uploadContext);
+      uploadLatencyMs = TimeUnit.NANOSECONDS.toMillis(uploadContext.stop());
       this.owningClient.uploadThroughput.mark(blob.length);
       this.owningClient.blobSizeHistogram.update(blob.length);
       this.owningClient.blobRowCountHistogram.update(
@@ -531,7 +538,12 @@ BlobMetadata upload(
     }
 
     return BlobMetadata.createBlobMetadata(
-        filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata, blobLatencies);
+        filePath,
+        BlobBuilder.computeMD5(blob),
+        bdecVersion,
+        metadata,
+        buildLatencyMs,
+        uploadLatencyMs);
   }
 
   /**

From 1fb23ceaa0fa277fc228cc48f0099d209c6ec5ff Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Wed, 5 Apr 2023 18:16:08 -0700
Subject: [PATCH 173/356] nanosecond test fix

---
 .../ingest/streaming/internal/FlushServiceTest.java        | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index fadda7979..cd69f04e7 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -35,6 +35,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TimeZone;
+import java.util.concurrent.TimeUnit;
 import javax.crypto.BadPaddingException;
 import javax.crypto.IllegalBlockSizeException;
 import javax.crypto.NoSuchPaddingException;
@@ -643,8 +644,8 @@ public void testBuildAndUpload() throws Exception {
     List<ChannelMetadata> channelMetadataResult = metadataResult.getChannels();
 
     Assert.assertEquals(BlobBuilder.computeMD5(blobCaptor.getValue()), blobMetadata.getMD5());
-    Assert.assertEquals(blobMetadata.getBuildLatencyMs(), expectedBuildLatencyMs);
-    Assert.assertEquals(blobMetadata.getUploadLatencyMs(), expectedUploadLatencyMs);
+    Assert.assertEquals(expectedBuildLatencyMs, blobMetadata.getBuildLatencyMs());
+    Assert.assertEquals(expectedUploadLatencyMs, blobMetadata.getUploadLatencyMs());
 
     Assert.assertEquals(
         expectedChunkEpInfo.getRowCount(), metadataResult.getEpInfo().getRowCount());
@@ -913,7 +914,7 @@ public void testEncryptionDecryption()
 
   private Timer setupTimer(long expectedLatencyMs) {
     Timer.Context timerContext = Mockito.mock(Timer.Context.class);
-    Mockito.when(timerContext.stop()).thenReturn(expectedLatencyMs);
+    Mockito.when(timerContext.stop()).thenReturn(TimeUnit.MILLISECONDS.toNanos(expectedLatencyMs));
     Timer timer = Mockito.mock(Timer.class);
     Mockito.when(timer.time()).thenReturn(timerContext);
 

From 0fb6ef6894f3e144f9375e55a4f8af8e6edc4a23 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Thu, 6 Apr 2023 15:46:13 -0700
Subject: [PATCH 174/356] add wait to flush test

---
 .../ingest/streaming/internal/FlushServiceTest.java        | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index cd69f04e7..d5e1202e0 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -464,6 +464,7 @@ public void testGetFilePath() {
   @Test
   public void testFlush() throws Exception {
     long beginTestTimestamp = System.currentTimeMillis();
+    TimeUnit.MILLISECONDS.sleep(100);
 
     TestContext<?> testContext = testContextFactory.create();
     FlushService<?> flushService = testContext.flushService;
@@ -481,7 +482,7 @@ public void testFlush() throws Exception {
     Mockito.verify(flushService, Mockito.times(1)).distributeFlushTasks();
     Mockito.verify(registerService, Mockito.times(1))
         .registerBlobs(
-            ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l > beginTestTimestamp));
+            ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l >= beginTestTimestamp));
 
     // isNeedFlush = true flushes
     flushService.isNeedFlush = true;
@@ -489,7 +490,7 @@ public void testFlush() throws Exception {
     Mockito.verify(flushService, Mockito.times(2)).distributeFlushTasks();
     Mockito.verify(registerService, Mockito.times(2))
         .registerBlobs(
-            ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l > beginTestTimestamp));
+            ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l >= beginTestTimestamp));
     Assert.assertFalse(flushService.isNeedFlush);
 
     // lastFlushTime causes flush
@@ -498,7 +499,7 @@ public void testFlush() throws Exception {
     Mockito.verify(flushService, Mockito.times(3)).distributeFlushTasks();
     Mockito.verify(registerService, Mockito.times(3))
         .registerBlobs(
-            ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l > beginTestTimestamp));
+            ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l >= beginTestTimestamp));
     Assert.assertTrue(flushService.lastFlushTime > 0);
   }
 

From 568a098357987992ad3d8cfb4d58f1d90fd5c665 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Tue, 11 Apr 2023 12:41:01 -0700
Subject: [PATCH 175/356] autoformatting

---
 .../streaming/internal/BlobLatencies.java     |  5 ++-
 .../streaming/internal/FlushService.java      | 37 +++++--------------
 .../streaming/internal/RegisterService.java   |  4 +-
 .../streaming/internal/FlushServiceTest.java  | 29 ++++-----------
 .../internal/RegisterServiceTest.java         | 12 ++----
 .../SnowflakeStreamingIngestClientTest.java   | 33 ++++++++---------
 6 files changed, 40 insertions(+), 80 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
index 171fef084..c4eaf2e7e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
@@ -6,6 +6,7 @@
 
 import com.codahale.metrics.Timer;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.concurrent.TimeUnit;
 
 /** Latency information for a blob */
 class BlobLatencies {
@@ -47,13 +48,13 @@ long getRegisterStartTimestamp() {
 
   void setBuildLatencyMs(Timer.Context buildLatencyContext) {
     if (buildLatencyContext != null) {
-      this.buildLatencyMs = buildLatencyContext.stop();
+      this.buildLatencyMs = TimeUnit.NANOSECONDS.toMillis(buildLatencyContext.stop());
     }
   }
 
   void setUploadLatencyMs(Timer.Context uploadLatencyContext) {
     if (uploadLatencyContext != null) {
-      this.uploadLatencyMs = uploadLatencyContext.stop();
+      this.uploadLatencyMs = TimeUnit.NANOSECONDS.toMillis(uploadLatencyContext.stop());
     }
   }
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index fee2ef20f..8b6990b39 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -134,13 +134,12 @@ List<List<ChannelData<T>>> getData() {
       SnowflakeStreamingIngestClientInternal<T> client,
       ChannelCache<T> cache,
       StreamingIngestStage targetStage, // For testing
-      RegisterService registerService,
       boolean isTestMode) {
     this.owningClient = client;
     this.channelCache = cache;
     this.targetStage = targetStage;
     this.counter = new AtomicLong(0);
-    this.registerService = registerService;
+    this.registerService = new RegisterService<>(client, isTestMode);
     this.isNeedFlush = false;
     this.lastFlushTime = System.currentTimeMillis();
     this.isTestMode = isTestMode;
@@ -240,13 +239,11 @@ private void logFlushTask(boolean isForce, long timeDiffMillis) {
   /**
    * Registers blobs with Snowflake
    *
-   * @param flushJobStartTime When the flush job began
    * @return
    */
-  private CompletableFuture<Void> registerFuture(long flushJobStartTime) {
+  private CompletableFuture<Void> registerFuture() {
     return CompletableFuture.runAsync(
-        () -> this.registerService.registerBlobs(latencyTimerContextMap, flushJobStartTime),
-        this.registerWorker);
+        () -> this.registerService.registerBlobs(latencyTimerContextMap), this.registerWorker);
   }
 
   /**
@@ -261,7 +258,6 @@ private CompletableFuture<Void> registerFuture(long flushJobStartTime) {
    */
   CompletableFuture<Void> flush(boolean isForce) {
     long timeDiffMillis = System.currentTimeMillis() - this.lastFlushTime;
-    long flushJobStartTime = System.currentTimeMillis();
 
     if (isForce
         || (!DISABLE_BACKGROUND_FLUSH
@@ -272,7 +268,7 @@ CompletableFuture<Void> flush(boolean isForce) {
 
       return this.statsFuture()
           .thenCompose((v) -> this.distributeFlush(isForce, timeDiffMillis))
-          .thenCompose((v) -> this.registerFuture(flushJobStartTime));
+          .thenCompose((v) -> this.registerFuture());
     }
     return this.statsFuture();
   }
@@ -493,14 +489,9 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
     // Construct the blob along with the metadata of the blob
     BlobBuilder.Blob blob = BlobBuilder.constructBlobAndMetadata(filePath, blobData, bdecVersion);
 
-    return buildContext != null
-        ? upload(
-            filePath,
-            blob.blobBytes,
-            blob.chunksMetadataList,
-            TimeUnit.NANOSECONDS.toMillis(buildContext.stop()))
-        : upload(
-            filePath, blob.blobBytes, blob.chunksMetadataList, BlobMetadata.DEFAULT_BLOB_LATENCY);
+    blob.blobLatencies.setBuildLatencyMs(buildContext);
+
+    return upload(filePath, blob.blobBytes, blob.chunksMetadataList, blob.blobLatencies);
   }
 
   /**
@@ -509,18 +500,15 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
    * @param filePath full path of the blob file
    * @param blob blob data
    * @param metadata a list of chunk metadata
-   * @param buildLatencyMs the build latency for the blob
    * @return BlobMetadata object used to create the register blob request
    */
   BlobMetadata upload(
-      String filePath, byte[] blob, List<ChunkMetadata> metadata, long buildLatencyMs)
+      String filePath, byte[] blob, List<ChunkMetadata> metadata, BlobLatencies blobLatencies)
       throws NoSuchAlgorithmException {
     logger.logInfo("Start uploading file={}, size={}", filePath, blob.length);
     long startTime = System.currentTimeMillis();
 
     Timer.Context uploadContext = Utils.createTimerContext(this.owningClient.uploadLatency);
-    long uploadLatencyMs = BlobMetadata.DEFAULT_BLOB_LATENCY;
-
     this.targetStage.put(filePath, blob);
 
     logger.logInfo(
@@ -530,7 +518,7 @@ BlobMetadata upload(
         System.currentTimeMillis() - startTime);
 
     if (uploadContext != null) {
-      uploadLatencyMs = TimeUnit.NANOSECONDS.toMillis(uploadContext.stop());
+      blobLatencies.setUploadLatencyMs(uploadContext);
       this.owningClient.uploadThroughput.mark(blob.length);
       this.owningClient.blobSizeHistogram.update(blob.length);
       this.owningClient.blobRowCountHistogram.update(
@@ -538,12 +526,7 @@ BlobMetadata upload(
     }
 
     return BlobMetadata.createBlobMetadata(
-        filePath,
-        BlobBuilder.computeMD5(blob),
-        bdecVersion,
-        metadata,
-        buildLatencyMs,
-        uploadLatencyMs);
+        filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata, blobLatencies);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index 151666656..3c2aefe08 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -79,11 +79,9 @@ void addBlobs(List<Pair<FlushService.BlobData<T>, CompletableFuture<BlobMetadata
    * the ordering is maintained across independent blobs in the same channel.
    *
    * @param latencyTimerContextMap the map that stores the latency timer for each blob
-   * @param flushJobStartTime When the flush job began
    * @return a list of blob names that have errors during registration
    */
-  List<FlushService.BlobData<T>> registerBlobs(
-      Map<String, Timer.Context> latencyTimerContextMap, long flushJobStartTime) {
+  List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyTimerContextMap) {
     List<FlushService.BlobData<T>> errorBlobs = new ArrayList<>();
     if (!this.blobsList.isEmpty()) {
       // Will skip and try again later if someone else is holding the lock
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index d5e1202e0..775d99673 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -105,8 +105,7 @@ private abstract static class TestContext<T> implements AutoCloseable {
       channelCache = new ChannelCache<>();
       Mockito.when(client.getChannelCache()).thenReturn(channelCache);
       registerService = Mockito.spy(new RegisterService(client, client.isTestMode()));
-      flushService =
-          Mockito.spy(new FlushService<>(client, channelCache, stage, registerService, false));
+      flushService = Mockito.spy(new FlushService<>(client, channelCache, stage, false));
     }
 
     ChannelData<T> flushChannel(String name) {
@@ -463,43 +462,28 @@ public void testGetFilePath() {
 
   @Test
   public void testFlush() throws Exception {
-    long beginTestTimestamp = System.currentTimeMillis();
-    TimeUnit.MILLISECONDS.sleep(100);
-
     TestContext<?> testContext = testContextFactory.create();
     FlushService<?> flushService = testContext.flushService;
-    RegisterService<?> registerService = testContext.registerService;
 
     // Nothing to flush
     flushService.flush(false).get();
     Mockito.verify(flushService, Mockito.times(0)).distributeFlushTasks();
-    Mockito.verify(registerService, Mockito.times(0))
-        .registerBlobs(ArgumentMatchers.anyMap(), ArgumentMatchers.anyLong());
 
     // Force = true flushes
     flushService.flush(true).get();
     Mockito.verify(flushService).distributeFlushTasks();
     Mockito.verify(flushService, Mockito.times(1)).distributeFlushTasks();
-    Mockito.verify(registerService, Mockito.times(1))
-        .registerBlobs(
-            ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l >= beginTestTimestamp));
 
     // isNeedFlush = true flushes
     flushService.isNeedFlush = true;
     flushService.flush(false).get();
     Mockito.verify(flushService, Mockito.times(2)).distributeFlushTasks();
-    Mockito.verify(registerService, Mockito.times(2))
-        .registerBlobs(
-            ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l >= beginTestTimestamp));
     Assert.assertFalse(flushService.isNeedFlush);
 
     // lastFlushTime causes flush
     flushService.lastFlushTime = 0;
     flushService.flush(false).get();
     Mockito.verify(flushService, Mockito.times(3)).distributeFlushTasks();
-    Mockito.verify(registerService, Mockito.times(3))
-        .registerBlobs(
-            ArgumentMatchers.anyMap(), ArgumentMatchers.longThat(l -> l >= beginTestTimestamp));
     Assert.assertTrue(flushService.lastFlushTime > 0);
   }
 
@@ -638,15 +622,17 @@ public void testBuildAndUpload() throws Exception {
             nameCaptor.capture(),
             blobCaptor.capture(),
             metadataCaptor.capture(),
-            ArgumentMatchers.anyLong());
+            ArgumentMatchers.any());
     Assert.assertEquals("file_name", nameCaptor.getValue());
 
     ChunkMetadata metadataResult = metadataCaptor.getValue().get(0);
     List<ChannelMetadata> channelMetadataResult = metadataResult.getChannels();
 
     Assert.assertEquals(BlobBuilder.computeMD5(blobCaptor.getValue()), blobMetadata.getMD5());
-    Assert.assertEquals(expectedBuildLatencyMs, blobMetadata.getBuildLatencyMs());
-    Assert.assertEquals(expectedUploadLatencyMs, blobMetadata.getUploadLatencyMs());
+    Assert.assertEquals(
+        expectedBuildLatencyMs, blobMetadata.getBlobLatencies().getBuildLatencyMs());
+    Assert.assertEquals(
+        expectedUploadLatencyMs, blobMetadata.getBlobLatencies().getUploadLatencyMs());
 
     Assert.assertEquals(
         expectedChunkEpInfo.getRowCount(), metadataResult.getEpInfo().getRowCount());
@@ -777,8 +763,7 @@ public void testInvalidateChannels() {
     StreamingIngestStage stage = Mockito.mock(StreamingIngestStage.class);
     Mockito.when(stage.getClientPrefix()).thenReturn("client_prefix");
     FlushService<StubChunkData> flushService =
-        new FlushService<>(
-            client, channelCache, stage, new RegisterService(client, client.isTestMode()), false);
+        new FlushService<>(client, channelCache, stage, false);
     flushService.invalidateAllChannelsInBlob(blobData);
 
     Assert.assertFalse(channel1.isValid());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
index d35ccf5bb..9638401d3 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
@@ -26,8 +26,7 @@ public void testRegisterService() {
             CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null, null)));
     rs.addBlobs(Collections.singletonList(blobFuture));
     Assert.assertEquals(1, rs.getBlobsList().size());
-    List<FlushService.BlobData<StubChunkData>> errorBlobs =
-        rs.registerBlobs(null, System.currentTimeMillis());
+    List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null);
     Assert.assertEquals(0, rs.getBlobsList().size());
     Assert.assertEquals(0, errorBlobs.size());
   }
@@ -58,8 +57,7 @@ public void testRegisterServiceTimeoutException() throws Exception {
     rs.addBlobs(Arrays.asList(blobFuture1, blobFuture2));
     Assert.assertEquals(2, rs.getBlobsList().size());
     try {
-      List<FlushService.BlobData<StubChunkData>> errorBlobs =
-          rs.registerBlobs(null, System.currentTimeMillis());
+      List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null);
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
       Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
@@ -95,8 +93,7 @@ public void testRegisterServiceTimeoutException_testRetries() throws Exception {
     rs.addBlobs(Arrays.asList(blobFuture1, blobFuture2));
     Assert.assertEquals(2, rs.getBlobsList().size());
     try {
-      List<FlushService.BlobData<StubChunkData>> errorBlobs =
-          rs.registerBlobs(null, System.currentTimeMillis());
+      List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null);
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
       Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
@@ -118,8 +115,7 @@ public void testRegisterServiceNonTimeoutException() {
     rs.addBlobs(Collections.singletonList(blobFuture));
     Assert.assertEquals(1, rs.getBlobsList().size());
     try {
-      List<FlushService.BlobData<StubChunkData>> errorBlobs =
-          rs.registerBlobs(null, System.currentTimeMillis());
+      List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null);
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
       Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index 4e162da69..e799e1b7c 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -68,7 +68,6 @@
 
 public class SnowflakeStreamingIngestClientTest {
   private static final ObjectMapper objectMapper = new ObjectMapper();
-  private static final long flushTimestamp = System.currentTimeMillis();
 
   SnowflakeStreamingIngestChannelInternal<StubChunkData> channel1;
   SnowflakeStreamingIngestChannelInternal<StubChunkData> channel2;
@@ -488,7 +487,7 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
 
     List<BlobMetadata> blobs =
         Collections.singletonList(
-            new BlobMetadata("path", "md5", Collections.singletonList(chunkMetadata)));
+            new BlobMetadata("path", "md5", Collections.singletonList(chunkMetadata), null));
 
     Map<Object, Object> payload = new HashMap<>();
     payload.put("request_id", null);
@@ -588,8 +587,8 @@ private Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> getRetryBlobMetadata(
     chunks1.add(chunkMetadata1);
     chunks1.add(chunkMetadata2);
     chunks2.add(chunkMetadata3);
-    blobs.add(new BlobMetadata("path1", "md51", chunks1));
-    blobs.add(new BlobMetadata("path2", "md52", chunks2));
+    blobs.add(new BlobMetadata("path1", "md51", chunks1, null));
+    blobs.add(new BlobMetadata("path2", "md52", chunks2, null));
 
     List<ChannelRegisterStatus> channelRegisterStatuses = new ArrayList<>();
     ChannelRegisterStatus status1 = new ChannelRegisterStatus();
@@ -674,8 +673,8 @@ public void testRegisterBlobErrorResponse() throws Exception {
 
     try {
       List<BlobMetadata> blobs =
-          Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
-      client.registerBlobs(blobs, flushTimestamp);
+          Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>(), null));
+      client.registerBlobs(blobs);
       Assert.fail("Register blob should fail on 404 error");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.REGISTER_BLOB_FAILURE.getMessageCode(), e.getVendorCode());
@@ -721,8 +720,8 @@ public void testRegisterBlobSnowflakeInternalErrorResponse() throws Exception {
 
     try {
       List<BlobMetadata> blobs =
-          Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
-      client.registerBlobs(blobs, flushTimestamp);
+          Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>(), null));
+      client.registerBlobs(blobs);
       Assert.fail("Register blob should fail on SF internal error");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.REGISTER_BLOB_FAILURE.getMessageCode(), e.getVendorCode());
@@ -776,8 +775,8 @@ public void testRegisterBlobSuccessResponse() throws Exception {
             null);
 
     List<BlobMetadata> blobs =
-        Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
-    client.registerBlobs(blobs, flushTimestamp);
+        Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>(), null));
+    client.registerBlobs(blobs);
   }
 
   @Test
@@ -863,10 +862,9 @@ public void testRegisterBlobsRetries() throws Exception {
     client.getChannelCache().addChannel(channel2);
     client.getChannelCache().addChannel(channel3);
     client.getChannelCache().addChannel(channel4);
-    client.registerBlobs(blobs, flushTimestamp);
+    client.registerBlobs(blobs);
     Mockito.verify(requestBuilder, Mockito.times(MAX_STREAMING_INGEST_API_CHANNEL_RETRY + 1))
-        .generateStreamingIngestPostRequest(
-            Mockito.contains(flushTimestamp + ""), Mockito.any(), Mockito.any());
+        .generateStreamingIngestPostRequest(Mockito.anyString(), Mockito.any(), Mockito.any());
     Assert.assertFalse(channel1.isValid());
     Assert.assertFalse(channel2.isValid());
   }
@@ -980,10 +978,9 @@ public void testRegisterBlobsRetriesSucceeds() throws Exception {
     client.getChannelCache().addChannel(channel3);
     client.getChannelCache().addChannel(channel4);
 
-    client.registerBlobs(blobs, flushTimestamp);
+    client.registerBlobs(blobs);
     Mockito.verify(requestBuilder, Mockito.times(2))
-        .generateStreamingIngestPostRequest(
-            Mockito.contains(flushTimestamp + ""), Mockito.any(), Mockito.any());
+        .generateStreamingIngestPostRequest(Mockito.anyString(), Mockito.any(), Mockito.any());
     Assert.assertTrue(channel1.isValid());
     Assert.assertTrue(channel2.isValid());
   }
@@ -1085,8 +1082,8 @@ public void testRegisterBlobResponseWithInvalidChannel() throws Exception {
     Assert.assertTrue(channel2.isValid());
 
     List<BlobMetadata> blobs =
-        Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>()));
-    client.registerBlobs(blobs, flushTimestamp);
+        Collections.singletonList(new BlobMetadata("path", "md5", new ArrayList<>(), null));
+    client.registerBlobs(blobs);
 
     // Channel2 should be invalidated now
     Assert.assertTrue(channel1.isValid());

From 1e0815486aa9b9d7ed7f00ffd263a51c2b47bda7 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Wed, 19 Apr 2023 16:34:41 -0700
Subject: [PATCH 176/356] nit renames

---
 .../streaming/internal/BlobLatencies.java     | 50 +++++++++----------
 .../streaming/internal/BlobMetadata.java      |  5 +-
 .../streaming/internal/FlushService.java      |  4 +-
 .../streaming/internal/RegisterService.java   | 18 ++++---
 .../internal/StreamingIngestUtils.java        | 35 ++++++-------
 .../streaming/internal/FlushServiceTest.java  |  4 +-
 6 files changed, 57 insertions(+), 59 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
index c4eaf2e7e..acef2543f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
@@ -10,59 +10,59 @@
 
 /** Latency information for a blob */
 class BlobLatencies {
-  public static final long DEFAULT_BLOB_LATENCY = -1;
+  public static final Long DEFAULT_BLOB_LATENCY = null;
 
-  private long buildLatencyMs;
-  private long uploadLatencyMs;
+  private Long buildDurationMs;
+  private Long uploadDurationMs;
 
-  private long flushStartTimestamp;
-  private long registerStartTimestamp;
+  private Long flushStartMs;
+  private Long registerStartMs;
 
   public BlobLatencies() {
-    this.buildLatencyMs = DEFAULT_BLOB_LATENCY;
-    this.uploadLatencyMs = DEFAULT_BLOB_LATENCY;
+    this.buildDurationMs = DEFAULT_BLOB_LATENCY;
+    this.uploadDurationMs = DEFAULT_BLOB_LATENCY;
 
-    this.flushStartTimestamp = DEFAULT_BLOB_LATENCY;
-    this.registerStartTimestamp = DEFAULT_BLOB_LATENCY;
+    this.flushStartMs = DEFAULT_BLOB_LATENCY;
+    this.registerStartMs = DEFAULT_BLOB_LATENCY;
   }
 
   @JsonProperty("build_latency_ms")
-  long getBuildLatencyMs() {
-    return this.buildLatencyMs;
+  long getBuildDurationMs() {
+    return this.buildDurationMs;
   }
 
   @JsonProperty("upload_latency_ms")
-  long getUploadLatencyMs() {
-    return this.uploadLatencyMs;
+  long getUploadDurationMs() {
+    return this.uploadDurationMs;
   }
 
   @JsonProperty("flush_start_timestamp")
-  long getFlushStartTimestamp() {
-    return this.flushStartTimestamp;
+  long getFlushStartMs() {
+    return this.flushStartMs;
   }
 
   @JsonProperty("register_start_timestamp")
-  long getRegisterStartTimestamp() {
-    return this.registerStartTimestamp;
+  long getRegisterStartMs() {
+    return this.registerStartMs;
   }
 
-  void setBuildLatencyMs(Timer.Context buildLatencyContext) {
+  void setBuildDurationMs(Timer.Context buildLatencyContext) {
     if (buildLatencyContext != null) {
-      this.buildLatencyMs = TimeUnit.NANOSECONDS.toMillis(buildLatencyContext.stop());
+      this.buildDurationMs = TimeUnit.NANOSECONDS.toMillis(buildLatencyContext.stop());
     }
   }
 
-  void setUploadLatencyMs(Timer.Context uploadLatencyContext) {
+  void setUploadDurationMs(Timer.Context uploadLatencyContext) {
     if (uploadLatencyContext != null) {
-      this.uploadLatencyMs = TimeUnit.NANOSECONDS.toMillis(uploadLatencyContext.stop());
+      this.uploadDurationMs = TimeUnit.NANOSECONDS.toMillis(uploadLatencyContext.stop());
     }
   }
 
-  void setFlushStartTimestamp(long flushStartTimestamp) {
-    this.flushStartTimestamp = flushStartTimestamp;
+  void setFlushStartMs(long flushStartMs) {
+    this.flushStartMs = flushStartMs;
   }
 
-  void setRegisterStartTimestamp(long registerStartTimestamp) {
-    this.registerStartTimestamp = registerStartTimestamp;
+  void setRegisterStartMs(long registerStartMs) {
+    this.registerStartMs = registerStartMs;
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index f7d44ae05..c04390402 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -6,9 +6,11 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import java.util.List;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ParameterProvider;
+import org.apache.arrow.util.VisibleForTesting;
+
+import java.util.List;
 
 /** Metadata for a blob that sends to Snowflake as part of the register blob request */
 class BlobMetadata {
@@ -19,6 +21,7 @@ class BlobMetadata {
   private final BlobLatencies blobLatencies;
 
   // used for testing only
+  @VisibleForTesting
   BlobMetadata(String path, String md5, List<ChunkMetadata> chunks, BlobLatencies blobLatencies) {
     this(path, md5, ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT, chunks, blobLatencies);
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 8b6990b39..a01f82e85 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -489,7 +489,7 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
     // Construct the blob along with the metadata of the blob
     BlobBuilder.Blob blob = BlobBuilder.constructBlobAndMetadata(filePath, blobData, bdecVersion);
 
-    blob.blobLatencies.setBuildLatencyMs(buildContext);
+    blob.blobLatencies.setBuildDurationMs(buildContext);
 
     return upload(filePath, blob.blobBytes, blob.chunksMetadataList, blob.blobLatencies);
   }
@@ -518,7 +518,7 @@ BlobMetadata upload(
         System.currentTimeMillis() - startTime);
 
     if (uploadContext != null) {
-      blobLatencies.setUploadLatencyMs(uploadContext);
+      blobLatencies.setUploadDurationMs(uploadContext);
       this.owningClient.uploadThroughput.mark(blob.length);
       this.owningClient.blobSizeHistogram.update(blob.length);
       this.owningClient.blobRowCountHistogram.update(
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index 3c2aefe08..1dd3584a2 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -4,10 +4,11 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
-
 import com.codahale.metrics.Timer;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.Utils;
+
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -17,9 +18,9 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.Utils;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
 /**
  * Register one or more blobs to the targeted Snowflake table, it will be done using the dedicated
@@ -197,8 +198,9 @@ List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyT
                 Utils.createTimerContext(this.owningClient.registerLatency);
 
             for (BlobMetadata blobMetadata : blobs) {
-              blobMetadata.getBlobLatencies().setRegisterStartTimestamp(System.currentTimeMillis());
-              blobMetadata.getBlobLatencies().setFlushStartTimestamp(System.currentTimeMillis());
+              long currentTimeMs = System.currentTimeMillis();
+              blobMetadata.getBlobLatencies().setRegisterStartMs(currentTimeMs);
+              blobMetadata.getBlobLatencies().setFlushStartMs(currentTimeMs);
             }
 
             // Register the blobs, and invalidate any channels that return a failure status code
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
index 5d5e10435..6116446c1 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
@@ -1,13 +1,7 @@
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
-
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import java.io.IOException;
-import java.util.Map;
-import java.util.function.Function;
 import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse;
 import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.ingest.connection.IngestResponseException;
@@ -17,6 +11,13 @@
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.SFException;
 
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Function;
+
+import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
+
 public class StreamingIngestUtils {
 
   private static class DefaultStatusGetter<T extends StreamingIngestResponse>
@@ -51,14 +52,14 @@ static <T extends StreamingIngestResponse> T executeWithRetries(
       CloseableHttpClient httpClient,
       RequestBuilder requestBuilder)
       throws IOException, IngestResponseException {
+    String payloadInString;
+    try {
+      payloadInString = objectMapper.writeValueAsString(payload);
+    } catch (JsonProcessingException e) {
+      throw new SFException(e, ErrorCode.BUILD_REQUEST_FAILURE, message);
+    }
     return executeWithRetries(
-        targetClass,
-        endpoint,
-        getPayloadAsStr(payload, message),
-        message,
-        apiName,
-        httpClient,
-        requestBuilder);
+            targetClass, endpoint, payloadInString, message, apiName, httpClient, requestBuilder);
   }
 
   static <T extends StreamingIngestResponse> T executeWithRetries(
@@ -122,12 +123,4 @@ public static String getShortname(final String fullname) {
     final String[] parts = fullname.split("/");
     return parts[parts.length - 1];
   }
-
-  public static String getPayloadAsStr(Map<Object, Object> payload, String message) {
-    try {
-      return objectMapper.writeValueAsString(payload);
-    } catch (JsonProcessingException e) {
-      throw new SFException(e, ErrorCode.BUILD_REQUEST_FAILURE, message);
-    }
-  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 775d99673..94d7cb320 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -630,9 +630,9 @@ public void testBuildAndUpload() throws Exception {
 
     Assert.assertEquals(BlobBuilder.computeMD5(blobCaptor.getValue()), blobMetadata.getMD5());
     Assert.assertEquals(
-        expectedBuildLatencyMs, blobMetadata.getBlobLatencies().getBuildLatencyMs());
+        expectedBuildLatencyMs, blobMetadata.getBlobLatencies().getBuildDurationMs());
     Assert.assertEquals(
-        expectedUploadLatencyMs, blobMetadata.getBlobLatencies().getUploadLatencyMs());
+        expectedUploadLatencyMs, blobMetadata.getBlobLatencies().getUploadDurationMs());
 
     Assert.assertEquals(
         expectedChunkEpInfo.getRowCount(), metadataResult.getEpInfo().getRowCount());

From b08fce9909b3cfa4185cf31725ce47386bc5c879 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Wed, 19 Apr 2023 16:34:44 -0700
Subject: [PATCH 177/356] autoformatting

---
 .../ingest/streaming/internal/BlobMetadata.java   |  3 +--
 .../streaming/internal/RegisterService.java       | 13 ++++++-------
 .../streaming/internal/StreamingIngestUtils.java  | 15 +++++++--------
 3 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index c04390402..7b4973b62 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -6,12 +6,11 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ParameterProvider;
 import org.apache.arrow.util.VisibleForTesting;
 
-import java.util.List;
-
 /** Metadata for a blob that sends to Snowflake as part of the register blob request */
 class BlobMetadata {
   private final String path;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index 1dd3584a2..d00d4705c 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -4,11 +4,10 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import com.codahale.metrics.Timer;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.Utils;
+import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
+import com.codahale.metrics.Timer;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -18,9 +17,9 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.Utils;
 
 /**
  * Register one or more blobs to the targeted Snowflake table, it will be done using the dedicated
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
index 6116446c1..88b406a62 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
@@ -1,7 +1,13 @@
 package net.snowflake.ingest.streaming.internal;
 
+import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Function;
 import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse;
 import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.ingest.connection.IngestResponseException;
@@ -11,13 +17,6 @@
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.SFException;
 
-import java.io.IOException;
-import java.util.Map;
-import java.util.function.Function;
-
-import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
-import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
-
 public class StreamingIngestUtils {
 
   private static class DefaultStatusGetter<T extends StreamingIngestResponse>
@@ -59,7 +58,7 @@ static <T extends StreamingIngestResponse> T executeWithRetries(
       throw new SFException(e, ErrorCode.BUILD_REQUEST_FAILURE, message);
     }
     return executeWithRetries(
-            targetClass, endpoint, payloadInString, message, apiName, httpClient, requestBuilder);
+        targetClass, endpoint, payloadInString, message, apiName, httpClient, requestBuilder);
   }
 
   static <T extends StreamingIngestResponse> T executeWithRetries(

From d4b4448a28f457564fdffc4a34b77ddc4fda5816 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Mon, 10 Apr 2023 13:11:28 -0700
Subject: [PATCH 178/356] add bot

---
 .github/workflows/cla_bot.yml | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 .github/workflows/cla_bot.yml

diff --git a/.github/workflows/cla_bot.yml b/.github/workflows/cla_bot.yml
new file mode 100644
index 000000000..bfd7609f0
--- /dev/null
+++ b/.github/workflows/cla_bot.yml
@@ -0,0 +1,24 @@
+name: "CLA Assistant"
+on:
+  issue_comment:
+    types: [created]
+  pull_request_target:
+    types: [opened,closed,synchronize]
+
+jobs:
+  CLAssistant:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "CLA Assistant"
+        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
+        uses: contributor-assistant/github-action/@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_BOT_TOKEN }}
+        with:
+          path-to-signatures: 'signatures/version1.json'
+          path-to-document: 'https://github.com/Snowflake-Labs/CLA/blob/main/README.md'
+          branch: 'main'
+          allowlist: 'dependabot[bot],github-actions,Jenkins User,sfc-gh-snyk-sca-sa'
+          remote-organization-name: 'snowflakedb'
+          remote-repository-name: 'cla-db'

From 85f93a303cd1fd766fd9fb6f92741ddef478834c Mon Sep 17 00:00:00 2001
From: Tyler Jones <59716821+sfc-gh-tjones@users.noreply.github.com>
Date: Thu, 6 Apr 2023 12:27:51 -0700
Subject: [PATCH 179/356] Upgrades Jettison further (#437)

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 50acbf777..9c74d467d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -212,7 +212,7 @@
       <dependency>
         <groupId>org.codehaus.jettison</groupId>
         <artifactId>jettison</artifactId>
-        <version>1.5.2</version>
+        <version>1.5.4</version>
       </dependency>
       <dependency>
         <groupId>org.codehaus.woodstox</groupId>

From 575fb361ef673b8c76da8ac960448c2f3ce808ea Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Sun, 9 Apr 2023 18:24:26 -0700
Subject: [PATCH 180/356] NO-SNOW: InsertRow perf improvements (#450)

Over the past few months, we keep adding data validation logic in order to make sure everything ingested is valid, some of the changes cause perf regressions, in this PR, we try to improve a few logic:

- Remove un-necessary Map that stores columnName - unquotedColName mapping
- Use a different way to validate utf8 which is more performant
- Fix a test failure what caused by server side change
---
 .../streaming/internal/ArrowRowBuffer.java       | 16 ++++++----------
 .../streaming/internal/DataValidationUtil.java   | 15 +++------------
 .../streaming/internal/ParquetRowBuffer.java     | 15 +++++----------
 .../streaming/internal/datatypes/StringsIT.java  |  2 +-
 4 files changed, 15 insertions(+), 33 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index c253014c0..31c0ef53e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -428,14 +428,9 @@ private float convertRowToArrow(
     // Create new empty stats just for the current row.
     Map<String, RowBufferStats> forkedStatsMap = new HashMap<>();
 
-    // We need to iterate twice over the row and over unquoted names, we store the value to avoid
-    // re-computation
-    Map<String, String> userInputToUnquotedColumnNameMap = new HashMap<>();
-
     for (Map.Entry<String, Object> entry : row.entrySet()) {
       rowBufferSize += 0.125; // 1/8 for null value bitmap
       String columnName = LiteralQuoteUtils.unquoteColumnName(entry.getKey());
-      userInputToUnquotedColumnNameMap.put(entry.getKey(), columnName);
       Object value = entry.getValue();
       Field field = this.fields.get(columnName);
       Utils.assertNotNull("Arrow column field", field);
@@ -737,12 +732,13 @@ private float convertRowToArrow(
 
     // All input values passed validation, iterate over the columns again and combine their existing
     // statistics with the forked statistics for the current row.
-    for (String userInputColumnName : row.keySet()) {
-      String columnName = userInputToUnquotedColumnNameMap.get(userInputColumnName);
-      RowBufferStats stats = statsMap.get(columnName);
-      RowBufferStats forkedStats = forkedStatsMap.get(columnName);
-      statsMap.put(columnName, RowBufferStats.getCombinedStats(stats, forkedStats));
+    for (Map.Entry<String, RowBufferStats> forkedColStats : forkedStatsMap.entrySet()) {
+      String columnName = forkedColStats.getKey();
+      statsMap.put(
+          columnName,
+          RowBufferStats.getCombinedStats(statsMap.get(columnName), forkedColStats.getValue()));
     }
+
     // Insert nulls to the columns that doesn't show up in the input
     for (String columnName : Sets.difference(this.fields.keySet(), inputColumnNames)) {
       rowBufferSize += 0.125; // 1/8 for null value bitmap
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index d9fd77273..3fac37429 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -13,10 +13,6 @@
 import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
 import java.math.BigDecimal;
 import java.math.BigInteger;
-import java.nio.CharBuffer;
-import java.nio.charset.CharacterCodingException;
-import java.nio.charset.CharsetEncoder;
-import java.nio.charset.CodingErrorAction;
 import java.nio.charset.StandardCharsets;
 import java.time.Instant;
 import java.time.LocalDate;
@@ -846,14 +842,9 @@ private static String sanitizeValueForExceptionMessage(Object value) {
    * UTF-16 surrogate, for example.
    */
   private static void verifyValidUtf8(String input, String columnName, String dataType) {
-    CharsetEncoder charsetEncoder =
-        StandardCharsets.UTF_8
-            .newEncoder()
-            .onMalformedInput(CodingErrorAction.REPORT)
-            .onUnmappableCharacter(CodingErrorAction.REPORT);
-    try {
-      charsetEncoder.encode(CharBuffer.wrap(input));
-    } catch (CharacterCodingException e) {
+    String roundTripStr =
+        new String(input.getBytes(StandardCharsets.UTF_8), StandardCharsets.UTF_8);
+    if (!input.equals(roundTripStr)) {
       throw valueFormatNotAllowedException(columnName, input, dataType, "Invalid Unicode string");
     }
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index bc53a61a7..9a37831e8 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -183,15 +183,10 @@ private float addRow(
     // Create new empty stats just for the current row.
     Map<String, RowBufferStats> forkedStatsMap = new HashMap<>();
 
-    // We need to iterate twice over the row and over unquoted names, we store the value to avoid
-    // re-computation
-    Map<String, String> userInputToUnquotedColumnNameMap = new HashMap<>();
-
     for (Map.Entry<String, Object> entry : row.entrySet()) {
       String key = entry.getKey();
       Object value = entry.getValue();
       String columnName = LiteralQuoteUtils.unquoteColumnName(key);
-      userInputToUnquotedColumnNameMap.put(key, columnName);
       int colIndex = fieldIndex.get(columnName).getSecond();
       RowBufferStats forkedStats = statsMap.get(columnName).forkEmpty();
       forkedStatsMap.put(columnName, forkedStats);
@@ -209,11 +204,11 @@ private float addRow(
 
     // All input values passed validation, iterate over the columns again and combine their existing
     // statistics with the forked statistics for the current row.
-    for (String userInputColumnName : row.keySet()) {
-      String columnName = userInputToUnquotedColumnNameMap.get(userInputColumnName);
-      RowBufferStats stats = statsMap.get(columnName);
-      RowBufferStats forkedStats = forkedStatsMap.get(columnName);
-      statsMap.put(columnName, RowBufferStats.getCombinedStats(stats, forkedStats));
+    for (Map.Entry<String, RowBufferStats> forkedColStats : forkedStatsMap.entrySet()) {
+      String columnName = forkedColStats.getKey();
+      statsMap.put(
+          columnName,
+          RowBufferStats.getCombinedStats(statsMap.get(columnName), forkedColStats.getValue()));
     }
 
     // Increment null count for column missing in the input map
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
index f41e24dc1..cfa59e0b1 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
@@ -217,7 +217,7 @@ public void testCollatedColumnsNotSupported() throws SQLException {
       openChannel(tableName);
       Assert.fail("Opening a channel shouldn't have succeeded");
     } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.UNSUPPORTED_DATA_TYPE.getMessageCode(), e.getVendorCode());
+      Assert.assertEquals(ErrorCode.OPEN_CHANNEL_FAILURE.getMessageCode(), e.getVendorCode());
     }
   }
 }

From 7b197c8db05fa448b6820c64568e6369a1f6c246 Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Tue, 11 Apr 2023 11:32:11 +0200
Subject: [PATCH 181/356] =?UTF-8?q?@no-snow=20print=20stack=20trace=20for?=
 =?UTF-8?q?=20failing=20test=20(StreamingIngestChannelTe=E2=80=A6=20(#451)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

@no-snow print stack trace for failing test (StreamingIngestChannelTest.testInsertRowThrottling) on windows
---
 .../streaming/internal/SnowflakeStreamingIngestChannelTest.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 28aca8d77..b810abadb 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -571,6 +571,7 @@ public void testInsertRowThrottling() {
       Assert.fail("the insert should be throttled.");
     } catch (TimeoutException ignored) {
     } catch (Exception e) {
+      e.printStackTrace();
       Assert.fail("unexpected exception encountered.");
     }
 
@@ -580,6 +581,7 @@ public void testInsertRowThrottling() {
     try {
       future.get(5L, TimeUnit.SECONDS);
     } catch (Exception e) {
+      e.printStackTrace();
       Assert.fail("unexpected exception encountered.");
     }
   }

From bc477231f36de69f0471b51352b1451633d206b0 Mon Sep 17 00:00:00 2001
From: Jay Patel <jay.patel@snowflake.com>
Date: Tue, 11 Apr 2023 11:49:01 -0700
Subject: [PATCH 182/356] SNOW-783754 Do not log column values in Exception +
 Pass in row Index in Error Message (#453)

* NO-SNOW Do not log column values in Exception

* Add new invalid row value error message, fix tests

* Pass in rowIndex in error message instead of Value

* revert local example change

* Refactoring bug - Pass 0 in tests and curRowIndex in actual function

* Stop printing big decimal values too

* Well written tests fails my tests, fixing IT test

* Rename to invalid_format_row

* Rename rowCount to bufferRowCount

* Rename curRowIndex to insertRowsCurrIndex and pass in that value instead of bufferedRowIndex
---
 .../streaming/internal/AbstractRowBuffer.java |  47 +-
 .../streaming/internal/ArrowRowBuffer.java    | 131 ++-
 .../internal/DataValidationUtil.java          | 218 ++--
 .../streaming/internal/ParquetRowBuffer.java  |  23 +-
 .../internal/ParquetValueParser.java          |  91 +-
 .../net/snowflake/ingest/utils/ErrorCode.java |   5 +-
 .../ingest/ingest_error_messages.properties   |   1 +
 .../internal/DataValidationUtilTest.java      | 978 ++++++++++--------
 .../internal/ParquetValueParserTest.java      |  48 +-
 .../streaming/internal/RowBufferTest.java     |  30 +-
 .../streaming/internal/StreamingIngestIT.java |   4 +-
 11 files changed, 895 insertions(+), 681 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index fa57199db..3c2c34831 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -147,8 +147,8 @@ public int getOrdinal() {
   // Lock used to protect the buffers from concurrent read/write
   private final Lock flushLock;
 
-  // Current row count
-  @VisibleForTesting volatile int rowCount;
+  // Current row count buffered
+  @VisibleForTesting volatile int bufferedRowCount;
 
   // Current buffer size
   private volatile float bufferSize;
@@ -188,7 +188,7 @@ public int getOrdinal() {
     this.allocator = allocator;
     this.nonNullableFieldNames = new HashSet<>();
     this.flushLock = new ReentrantLock();
-    this.rowCount = 0;
+    this.bufferedRowCount = 0;
     this.bufferSize = 0F;
 
     // Initialize empty stats
@@ -258,7 +258,7 @@ Set<String> verifyInputColumns(
         error.setExtraColNames(extraCols);
       }
       throw new SFException(
-          ErrorCode.INVALID_ROW,
+          ErrorCode.INVALID_FORMAT_ROW,
           "Extra columns: " + extraCols,
           "Columns not present in the table shouldn't be specified.");
     }
@@ -276,7 +276,7 @@ Set<String> verifyInputColumns(
         error.setMissingNotNullColNames(missingCols);
       }
       throw new SFException(
-          ErrorCode.INVALID_ROW,
+          ErrorCode.INVALID_FORMAT_ROW,
           "Missing columns: " + missingCols,
           "Values for all non-nullable columns must be specified.");
     }
@@ -301,7 +301,7 @@ public InsertValidationResponse insertRows(
     InsertValidationResponse response = new InsertValidationResponse();
     this.flushLock.lock();
     try {
-      this.channelState.updateInsertStats(System.currentTimeMillis(), this.rowCount);
+      this.channelState.updateInsertStats(System.currentTimeMillis(), this.bufferedRowCount);
       if (onErrorOption == OpenChannelRequest.OnErrorOption.CONTINUE) {
         // Used to map incoming row(nth row) to InsertError(for nth row) in response
         long rowIndex = 0;
@@ -310,8 +310,9 @@ public InsertValidationResponse insertRows(
               new InsertValidationResponse.InsertError(row, rowIndex);
           try {
             Set<String> inputColumnNames = verifyInputColumns(row, error);
-            rowsSizeInBytes += addRow(row, this.rowCount, this.statsMap, inputColumnNames);
-            this.rowCount++;
+            rowsSizeInBytes +=
+                addRow(row, this.bufferedRowCount, this.statsMap, inputColumnNames, rowIndex);
+            this.bufferedRowCount++;
           } catch (SFException e) {
             error.setException(e);
             response.addError(error);
@@ -321,7 +322,7 @@ public InsertValidationResponse insertRows(
             response.addError(error);
           }
           rowIndex++;
-          if (this.rowCount == Integer.MAX_VALUE) {
+          if (this.bufferedRowCount == Integer.MAX_VALUE) {
             throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
           }
         }
@@ -331,17 +332,18 @@ public InsertValidationResponse insertRows(
         int tempRowCount = 0;
         for (Map<String, Object> row : rows) {
           Set<String> inputColumnNames = verifyInputColumns(row, null);
-          tempRowsSizeInBytes += addTempRow(row, tempRowCount, this.tempStatsMap, inputColumnNames);
+          tempRowsSizeInBytes +=
+              addTempRow(row, tempRowCount, this.tempStatsMap, inputColumnNames, tempRowCount);
           tempRowCount++;
         }
 
         moveTempRowsToActualBuffer(tempRowCount);
 
         rowsSizeInBytes = tempRowsSizeInBytes;
-        if ((long) this.rowCount + tempRowCount >= Integer.MAX_VALUE) {
+        if ((long) this.bufferedRowCount + tempRowCount >= Integer.MAX_VALUE) {
           throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
         }
-        this.rowCount += tempRowCount;
+        this.bufferedRowCount += tempRowCount;
         this.statsMap.forEach(
             (colName, stats) ->
                 this.statsMap.put(
@@ -371,7 +373,7 @@ public InsertValidationResponse insertRows(
   @Override
   public ChannelData<T> flush(final String filePath) {
     logger.logDebug("Start get data for channel={}", channelFullyQualifiedName);
-    if (this.rowCount > 0) {
+    if (this.bufferedRowCount > 0) {
       Optional<T> oldData = Optional.empty();
       int oldRowCount = 0;
       float oldBufferSize = 0F;
@@ -384,10 +386,10 @@ public ChannelData<T> flush(final String filePath) {
 
       this.flushLock.lock();
       try {
-        if (this.rowCount > 0) {
+        if (this.bufferedRowCount > 0) {
           // Transfer the ownership of the vectors
           oldData = getSnapshot(filePath);
-          oldRowCount = this.rowCount;
+          oldRowCount = this.bufferedRowCount;
           oldBufferSize = this.bufferSize;
           oldRowSequencer = this.channelState.incrementAndGetRowSequencer();
           oldOffsetToken = this.channelState.getOffsetToken();
@@ -431,16 +433,19 @@ public ChannelData<T> flush(final String filePath) {
    * Add an input row to the buffer.
    *
    * @param row input row
-   * @param curRowIndex current row index to use
+   * @param bufferedRowIndex Row number of buffered rows which will eventually by flushed.
    * @param statsMap column stats map
    * @param formattedInputColumnNames list of input column names after formatting
+   * @param insertRowIndex Index of the rows given in insertRows API. Not the same as
+   *     bufferedRowIndex
    * @return row size
    */
   abstract float addRow(
       Map<String, Object> row,
-      int curRowIndex,
+      int bufferedRowIndex,
       Map<String, RowBufferStats> statsMap,
-      Set<String> formattedInputColumnNames);
+      Set<String> formattedInputColumnNames,
+      final long insertRowIndex);
 
   /**
    * Add an input row to the temporary row buffer.
@@ -452,13 +457,15 @@ abstract float addRow(
    * @param curRowIndex current row index to use
    * @param statsMap column stats map
    * @param formattedInputColumnNames list of input column names after formatting
+   * @param insertRowIndex index of the row being inserteed from User Input List
    * @return row size
    */
   abstract float addTempRow(
       Map<String, Object> row,
       int curRowIndex,
       Map<String, RowBufferStats> statsMap,
-      Set<String> formattedInputColumnNames);
+      Set<String> formattedInputColumnNames,
+      long insertRowIndex);
 
   /** Move rows from the temporary buffer to the current row buffer. */
   abstract void moveTempRowsToActualBuffer(int tempRowCount);
@@ -471,7 +478,7 @@ abstract float addTempRow(
 
   /** Reset the variables after each flush. Note that the caller needs to handle synchronization. */
   void reset() {
-    this.rowCount = 0;
+    this.bufferedRowCount = 0;
     this.bufferSize = 0F;
     this.statsMap.replaceAll((key, value) -> value.forkEmpty());
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
index 31c0ef53e..3e936233d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
@@ -335,7 +335,7 @@ void moveTempRowsToActualBuffer(int tempRowCount) {
       FieldVector from = this.tempVectorsRoot.getVector(field);
       FieldVector to = this.vectorsRoot.getVector(field);
       for (int rowIdx = 0; rowIdx < tempRowCount; rowIdx++) {
-        to.copyFromSafe(rowIdx, this.rowCount + rowIdx, from);
+        to.copyFromSafe(rowIdx, this.bufferedRowCount + rowIdx, from);
       }
     }
   }
@@ -354,7 +354,7 @@ boolean hasColumns() {
   Optional<VectorSchemaRoot> getSnapshot(final String filePath) {
     List<FieldVector> oldVectors = new ArrayList<>();
     for (FieldVector vector : this.vectorsRoot.getFieldVectors()) {
-      vector.setValueCount(this.rowCount);
+      vector.setValueCount(this.bufferedRowCount);
       if (vector instanceof DecimalVector) {
         // DecimalVectors do not transfer FieldType metadata when using
         // vector.getTransferPair. We need to explicitly create the new vector to transfer to
@@ -379,7 +379,7 @@ Optional<VectorSchemaRoot> getSnapshot(final String filePath) {
       }
     }
     VectorSchemaRoot root = new VectorSchemaRoot(oldVectors);
-    root.setRowCount(this.rowCount);
+    root.setRowCount(this.bufferedRowCount);
     return oldVectors.isEmpty() ? Optional.empty() : Optional.of(root);
   }
 
@@ -391,10 +391,12 @@ boolean hasColumn(String name) {
   @Override
   float addRow(
       Map<String, Object> row,
-      int curRowIndex,
+      int bufferedRowIndex,
       Map<String, RowBufferStats> statsMap,
-      Set<String> formattedInputColumnNames) {
-    return convertRowToArrow(row, vectorsRoot, curRowIndex, statsMap, formattedInputColumnNames);
+      Set<String> formattedInputColumnNames,
+      final long insertRowIndex) {
+    return convertRowToArrow(
+        row, vectorsRoot, bufferedRowIndex, statsMap, formattedInputColumnNames, insertRowIndex);
   }
 
   @Override
@@ -402,9 +404,10 @@ float addTempRow(
       Map<String, Object> row,
       int curRowIndex,
       Map<String, RowBufferStats> statsMap,
-      Set<String> formattedInputColumnNames) {
+      Set<String> formattedInputColumnNames,
+      long insertRowIndex) {
     return convertRowToArrow(
-        row, tempVectorsRoot, curRowIndex, statsMap, formattedInputColumnNames);
+        row, tempVectorsRoot, curRowIndex, statsMap, formattedInputColumnNames, insertRowIndex);
   }
 
   /**
@@ -412,17 +415,21 @@ float addTempRow(
    *
    * @param row input row
    * @param sourceVectors vectors (buffers) that hold the row
-   * @param curRowIndex current row index to use
+   * @param bufferedRowIndex Buffered row index. This is not the same as the input row index
    * @param statsMap column stats map
    * @param inputColumnNames list of input column names after formatting
+   * @param insertRowsCurrIndex Row index of the input Rows passed in {@link
+   *     net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel#insertRows(Iterable,
+   *     String)}
    * @return row size
    */
   private float convertRowToArrow(
       Map<String, Object> row,
       VectorSchemaRoot sourceVectors,
-      int curRowIndex,
+      int bufferedRowIndex,
       Map<String, RowBufferStats> statsMap,
-      Set<String> inputColumnNames) {
+      Set<String> inputColumnNames,
+      long insertRowsCurrIndex) {
     // Insert values to the corresponding arrow buffers
     float rowBufferSize = 0F;
     // Create new empty stats just for the current row.
@@ -452,36 +459,39 @@ private float convertRowToArrow(
             int columnScale = getColumnScale(field.getMetadata());
             BigDecimal inputAsBigDecimal =
                 DataValidationUtil.validateAndParseBigDecimal(
-                    forkedStats.getColumnDisplayName(), value);
+                    forkedStats.getColumnDisplayName(), value, insertRowsCurrIndex);
             // vector.setSafe requires the BigDecimal input scale explicitly match its scale
             inputAsBigDecimal = inputAsBigDecimal.setScale(columnScale, RoundingMode.HALF_UP);
 
-            DataValidationUtil.checkValueInRange(inputAsBigDecimal, columnScale, columnPrecision);
+            DataValidationUtil.checkValueInRange(
+                inputAsBigDecimal, columnScale, columnPrecision, insertRowsCurrIndex);
 
             if (columnScale != 0 || physicalType == ColumnPhysicalType.SB16) {
-              ((DecimalVector) vector).setSafe(curRowIndex, inputAsBigDecimal);
+              ((DecimalVector) vector).setSafe(bufferedRowIndex, inputAsBigDecimal);
               forkedStats.addIntValue(inputAsBigDecimal.unscaledValue());
               rowBufferSize += 16;
             } else {
               switch (physicalType) {
                 case SB1:
-                  ((TinyIntVector) vector).setSafe(curRowIndex, inputAsBigDecimal.byteValueExact());
+                  ((TinyIntVector) vector)
+                      .setSafe(bufferedRowIndex, inputAsBigDecimal.byteValueExact());
                   forkedStats.addIntValue(inputAsBigDecimal.toBigInteger());
                   rowBufferSize += 1;
                   break;
                 case SB2:
                   ((SmallIntVector) vector)
-                      .setSafe(curRowIndex, inputAsBigDecimal.shortValueExact());
+                      .setSafe(bufferedRowIndex, inputAsBigDecimal.shortValueExact());
                   forkedStats.addIntValue(inputAsBigDecimal.toBigInteger());
                   rowBufferSize += 2;
                   break;
                 case SB4:
-                  ((IntVector) vector).setSafe(curRowIndex, inputAsBigDecimal.intValueExact());
+                  ((IntVector) vector).setSafe(bufferedRowIndex, inputAsBigDecimal.intValueExact());
                   forkedStats.addIntValue(inputAsBigDecimal.toBigInteger());
                   rowBufferSize += 4;
                   break;
                 case SB8:
-                  ((BigIntVector) vector).setSafe(curRowIndex, inputAsBigDecimal.longValueExact());
+                  ((BigIntVector) vector)
+                      .setSafe(bufferedRowIndex, inputAsBigDecimal.longValueExact());
                   forkedStats.addIntValue(inputAsBigDecimal.toBigInteger());
                   rowBufferSize += 8;
                   break;
@@ -499,9 +509,10 @@ private float convertRowToArrow(
                   DataValidationUtil.validateAndParseString(
                       forkedStats.getColumnDisplayName(),
                       value,
-                      Optional.ofNullable(maxLengthString).map(Integer::parseInt));
+                      Optional.ofNullable(maxLengthString).map(Integer::parseInt),
+                      insertRowsCurrIndex);
               Text text = new Text(str);
-              ((VarCharVector) vector).setSafe(curRowIndex, text);
+              ((VarCharVector) vector).setSafe(bufferedRowIndex, text);
               forkedStats.addStrValue(str);
               rowBufferSize += text.getBytes().length;
               break;
@@ -510,9 +521,9 @@ private float convertRowToArrow(
             {
               String str =
                   DataValidationUtil.validateAndParseObject(
-                      forkedStats.getColumnDisplayName(), value);
+                      forkedStats.getColumnDisplayName(), value, insertRowsCurrIndex);
               Text text = new Text(str);
-              ((VarCharVector) vector).setSafe(curRowIndex, text);
+              ((VarCharVector) vector).setSafe(bufferedRowIndex, text);
               rowBufferSize += text.getBytes().length;
               break;
             }
@@ -520,9 +531,9 @@ private float convertRowToArrow(
             {
               String str =
                   DataValidationUtil.validateAndParseArray(
-                      forkedStats.getColumnDisplayName(), value);
+                      forkedStats.getColumnDisplayName(), value, insertRowsCurrIndex);
               Text text = new Text(str);
-              ((VarCharVector) vector).setSafe(curRowIndex, text);
+              ((VarCharVector) vector).setSafe(bufferedRowIndex, text);
               rowBufferSize += text.getBytes().length;
               break;
             }
@@ -530,10 +541,10 @@ private float convertRowToArrow(
             {
               String str =
                   DataValidationUtil.validateAndParseVariant(
-                      forkedStats.getColumnDisplayName(), value);
+                      forkedStats.getColumnDisplayName(), value, insertRowsCurrIndex);
               if (str != null) {
                 Text text = new Text(str);
-                ((VarCharVector) vector).setSafe(curRowIndex, text);
+                ((VarCharVector) vector).setSafe(bufferedRowIndex, text);
                 rowBufferSize += text.getBytes().length;
               } else {
                 isParsedValueNull = true;
@@ -554,9 +565,10 @@ private float convertRowToArrow(
                           value,
                           getColumnScale(field.getMetadata()),
                           defaultTimezone,
-                          trimTimezone);
+                          trimTimezone,
+                          insertRowsCurrIndex);
                   BigInteger timestampBinary = timestampWrapper.toBinary(false);
-                  bigIntVector.setSafe(curRowIndex, timestampBinary.longValue());
+                  bigIntVector.setSafe(bufferedRowIndex, timestampBinary.longValue());
                   forkedStats.addIntValue(timestampBinary);
                   rowBufferSize += 8;
                   break;
@@ -569,7 +581,7 @@ private float convertRowToArrow(
                   IntVector fractionVector =
                       (IntVector) structVector.getChild(FIELD_FRACTION_IN_NANOSECONDS);
                   rowBufferSize += 0.25; // for children vector's null value
-                  structVector.setIndexDefined(curRowIndex);
+                  structVector.setIndexDefined(bufferedRowIndex);
 
                   TimestampWrapper timestampWrapper =
                       DataValidationUtil.validateAndParseTimestamp(
@@ -577,9 +589,10 @@ private float convertRowToArrow(
                           value,
                           getColumnScale(field.getMetadata()),
                           defaultTimezone,
-                          trimTimezone);
-                  epochVector.setSafe(curRowIndex, timestampWrapper.getEpoch());
-                  fractionVector.setSafe(curRowIndex, timestampWrapper.getFraction());
+                          trimTimezone,
+                          insertRowsCurrIndex);
+                  epochVector.setSafe(bufferedRowIndex, timestampWrapper.getEpoch());
+                  fractionVector.setSafe(bufferedRowIndex, timestampWrapper.getFraction());
                   rowBufferSize += 12;
                   forkedStats.addIntValue(timestampWrapper.toBinary(false));
                   break;
@@ -598,7 +611,7 @@ private float convertRowToArrow(
                   IntVector timezoneVector = (IntVector) structVector.getChild(FIELD_TIME_ZONE);
 
                   rowBufferSize += 0.25; // for children vector's null value
-                  structVector.setIndexDefined(curRowIndex);
+                  structVector.setIndexDefined(bufferedRowIndex);
 
                   TimestampWrapper timestampWrapper =
                       DataValidationUtil.validateAndParseTimestamp(
@@ -606,10 +619,11 @@ private float convertRowToArrow(
                           value,
                           getColumnScale(field.getMetadata()),
                           defaultTimezone,
-                          false);
+                          false,
+                          insertRowsCurrIndex);
                   epochVector.setSafe(
-                      curRowIndex, timestampWrapper.toBinary(false).longValueExact());
-                  timezoneVector.setSafe(curRowIndex, timestampWrapper.getTimeZoneIndex());
+                      bufferedRowIndex, timestampWrapper.toBinary(false).longValueExact());
+                  timezoneVector.setSafe(bufferedRowIndex, timestampWrapper.getTimeZoneIndex());
                   rowBufferSize += 12;
                   forkedStats.addIntValue(timestampWrapper.toBinary(true));
                   break;
@@ -624,7 +638,7 @@ private float convertRowToArrow(
                   IntVector timezoneVector = (IntVector) structVector.getChild(FIELD_TIME_ZONE);
 
                   rowBufferSize += 0.375; // for children vector's null value
-                  structVector.setIndexDefined(curRowIndex);
+                  structVector.setIndexDefined(bufferedRowIndex);
 
                   TimestampWrapper timestampWrapper =
                       DataValidationUtil.validateAndParseTimestamp(
@@ -632,10 +646,11 @@ private float convertRowToArrow(
                           value,
                           getColumnScale(field.getMetadata()),
                           defaultTimezone,
-                          false);
-                  epochVector.setSafe(curRowIndex, timestampWrapper.getEpoch());
-                  fractionVector.setSafe(curRowIndex, timestampWrapper.getFraction());
-                  timezoneVector.setSafe(curRowIndex, timestampWrapper.getTimeZoneIndex());
+                          false,
+                          insertRowsCurrIndex);
+                  epochVector.setSafe(bufferedRowIndex, timestampWrapper.getEpoch());
+                  fractionVector.setSafe(bufferedRowIndex, timestampWrapper.getFraction());
+                  timezoneVector.setSafe(bufferedRowIndex, timestampWrapper.getTimeZoneIndex());
                   rowBufferSize += 16;
                   BigInteger timeInBinary = timestampWrapper.toBinary(true);
                   forkedStats.addIntValue(timeInBinary);
@@ -651,8 +666,8 @@ private float convertRowToArrow(
               // Expect days past the epoch
               int intValue =
                   DataValidationUtil.validateAndParseDate(
-                      forkedStats.getColumnDisplayName(), value);
-              dateDayVector.setSafe(curRowIndex, intValue);
+                      forkedStats.getColumnDisplayName(), value, insertRowsCurrIndex);
+              dateDayVector.setSafe(bufferedRowIndex, intValue);
               forkedStats.addIntValue(BigInteger.valueOf(intValue));
               rowBufferSize += 4;
               break;
@@ -665,8 +680,9 @@ private float convertRowToArrow(
                       DataValidationUtil.validateAndParseTime(
                           forkedStats.getColumnDisplayName(),
                           value,
-                          getColumnScale(field.getMetadata()));
-                  ((IntVector) vector).setSafe(curRowIndex, timeInScale.intValue());
+                          getColumnScale(field.getMetadata()),
+                          insertRowsCurrIndex);
+                  ((IntVector) vector).setSafe(bufferedRowIndex, timeInScale.intValue());
                   forkedStats.addIntValue(timeInScale);
                   rowBufferSize += 4;
                   break;
@@ -677,8 +693,9 @@ private float convertRowToArrow(
                       DataValidationUtil.validateAndParseTime(
                           forkedStats.getColumnDisplayName(),
                           value,
-                          getColumnScale(field.getMetadata()));
-                  ((BigIntVector) vector).setSafe(curRowIndex, timeInScale.longValue());
+                          getColumnScale(field.getMetadata()),
+                          insertRowsCurrIndex);
+                  ((BigIntVector) vector).setSafe(bufferedRowIndex, timeInScale.longValue());
                   forkedStats.addIntValue(timeInScale);
                   rowBufferSize += 8;
                   break;
@@ -691,8 +708,8 @@ private float convertRowToArrow(
             {
               int intValue =
                   DataValidationUtil.validateAndParseBoolean(
-                      forkedStats.getColumnDisplayName(), value);
-              ((BitVector) vector).setSafe(curRowIndex, intValue);
+                      forkedStats.getColumnDisplayName(), value, insertRowsCurrIndex);
+              ((BitVector) vector).setSafe(bufferedRowIndex, intValue);
               rowBufferSize += 0.125;
               forkedStats.addIntValue(BigInteger.valueOf(intValue));
               break;
@@ -703,15 +720,17 @@ private float convertRowToArrow(
                 DataValidationUtil.validateAndParseBinary(
                     forkedStats.getColumnDisplayName(),
                     value,
-                    Optional.ofNullable(maxLengthString).map(Integer::parseInt));
-            ((VarBinaryVector) vector).setSafe(curRowIndex, bytes);
+                    Optional.ofNullable(maxLengthString).map(Integer::parseInt),
+                    insertRowsCurrIndex);
+            ((VarBinaryVector) vector).setSafe(bufferedRowIndex, bytes);
             forkedStats.addBinaryValue(bytes);
             rowBufferSize += bytes.length;
             break;
           case REAL:
             double doubleValue =
-                DataValidationUtil.validateAndParseReal(forkedStats.getColumnDisplayName(), value);
-            ((Float8Vector) vector).setSafe(curRowIndex, doubleValue);
+                DataValidationUtil.validateAndParseReal(
+                    forkedStats.getColumnDisplayName(), value, insertRowsCurrIndex);
+            ((Float8Vector) vector).setSafe(bufferedRowIndex, doubleValue);
             forkedStats.addRealValue(doubleValue);
             rowBufferSize += 8;
             break;
@@ -723,9 +742,9 @@ private float convertRowToArrow(
       if (value == null || isParsedValueNull) {
         if (!field.getFieldType().isNullable()) {
           throw new SFException(
-              ErrorCode.INVALID_ROW, columnName, "Passed null to non nullable field");
+              ErrorCode.INVALID_FORMAT_ROW, columnName, "Passed null to non nullable field");
         } else {
-          insertNull(vector, forkedStats, curRowIndex);
+          insertNull(vector, forkedStats, bufferedRowIndex);
         }
       }
     }
@@ -745,7 +764,7 @@ private float convertRowToArrow(
       insertNull(
           sourceVectors.getVector(this.fields.get(columnName)),
           statsMap.get(columnName),
-          curRowIndex);
+          bufferedRowIndex);
     }
 
     return rowBufferSize;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 3fac37429..d04f9d3fc 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -94,14 +94,15 @@ class DataValidationUtil {
    * @return JSON tree representing the input
    */
   private static JsonNode validateAndParseSemiStructuredAsJsonTree(
-      String columnName, Object input, String snowflakeType) {
+      String columnName, Object input, String snowflakeType, final long insertRowIndex) {
     if (input instanceof String) {
       String stringInput = (String) input;
-      verifyValidUtf8(stringInput, columnName, snowflakeType);
+      verifyValidUtf8(stringInput, columnName, snowflakeType, insertRowIndex);
       try {
         return objectMapper.readTree(stringInput);
       } catch (JsonProcessingException e) {
-        throw valueFormatNotAllowedException(columnName, input, snowflakeType, "Not a valid JSON");
+        throw valueFormatNotAllowedException(
+            columnName, snowflakeType, "Not a valid JSON", insertRowIndex);
       }
     } else if (isAllowedSemiStructuredType(input)) {
       return objectMapper.valueToTree(input);
@@ -118,7 +119,8 @@ private static JsonNode validateAndParseSemiStructuredAsJsonTree(
           "List<T>",
           "Map<String, T>",
           "T[]"
-        });
+        },
+        insertRowIndex);
   }
 
   /**
@@ -126,10 +128,12 @@ private static JsonNode validateAndParseSemiStructuredAsJsonTree(
    * see {@link DataValidationUtil#isAllowedSemiStructuredType}.
    *
    * @param input Object to validate
+   * @param insertRowIndex
    * @return JSON string representing the input
    */
-  static String validateAndParseVariant(String columnName, Object input) {
-    JsonNode node = validateAndParseSemiStructuredAsJsonTree(columnName, input, "VARIANT");
+  static String validateAndParseVariant(String columnName, Object input, long insertRowIndex) {
+    JsonNode node =
+        validateAndParseSemiStructuredAsJsonTree(columnName, input, "VARIANT", insertRowIndex);
 
     // Missing nodes are not valid json, ingest them as NULL instead
     if (node.isMissingNode()) {
@@ -141,11 +145,10 @@ static String validateAndParseVariant(String columnName, Object input) {
     if (stringLength > MAX_SEMI_STRUCTURED_LENGTH) {
       throw valueFormatNotAllowedException(
           columnName,
-          output,
           "VARIANT",
           String.format(
-              "Variant too long: length=%d maxLength=%d",
-              stringLength, MAX_SEMI_STRUCTURED_LENGTH));
+              "Variant too long: length=%d maxLength=%d", stringLength, MAX_SEMI_STRUCTURED_LENGTH),
+          insertRowIndex);
     }
     return output;
   }
@@ -258,10 +261,12 @@ static boolean isAllowedSemiStructuredType(Object o) {
    * DataValidationUtil#isAllowedSemiStructuredType}.
    *
    * @param input Object to validate
+   * @param insertRowIndex
    * @return JSON array representing the input
    */
-  static String validateAndParseArray(String columnName, Object input) {
-    JsonNode jsonNode = validateAndParseSemiStructuredAsJsonTree(columnName, input, "ARRAY");
+  static String validateAndParseArray(String columnName, Object input, long insertRowIndex) {
+    JsonNode jsonNode =
+        validateAndParseSemiStructuredAsJsonTree(columnName, input, "ARRAY", insertRowIndex);
 
     // Non-array values are ingested as single-element arrays, mimicking the Worksheets behavior
     if (!jsonNode.isArray()) {
@@ -274,10 +279,10 @@ static String validateAndParseArray(String columnName, Object input) {
     if (stringLength > MAX_SEMI_STRUCTURED_LENGTH) {
       throw valueFormatNotAllowedException(
           columnName,
-          output,
           "ARRAY",
           String.format(
-              "Array too large. length=%d maxLength=%d", stringLength, MAX_SEMI_STRUCTURED_LENGTH));
+              "Array too large. length=%d maxLength=%d", stringLength, MAX_SEMI_STRUCTURED_LENGTH),
+          insertRowIndex);
     }
     return output;
   }
@@ -288,12 +293,14 @@ static String validateAndParseArray(String columnName, Object input) {
    * see {@link DataValidationUtil#isAllowedSemiStructuredType}.
    *
    * @param input Object to validate
+   * @param insertRowIndex
    * @return JSON object representing the input
    */
-  static String validateAndParseObject(String columnName, Object input) {
-    JsonNode jsonNode = validateAndParseSemiStructuredAsJsonTree(columnName, input, "OBJECT");
+  static String validateAndParseObject(String columnName, Object input, long insertRowIndex) {
+    JsonNode jsonNode =
+        validateAndParseSemiStructuredAsJsonTree(columnName, input, "OBJECT", insertRowIndex);
     if (!jsonNode.isObject()) {
-      throw valueFormatNotAllowedException(columnName, jsonNode, "OBJECT", "Not an object");
+      throw valueFormatNotAllowedException(columnName, "OBJECT", "Not an object", insertRowIndex);
     }
 
     String output = jsonNode.toString();
@@ -302,11 +309,10 @@ static String validateAndParseObject(String columnName, Object input) {
     if (stringLength > MAX_SEMI_STRUCTURED_LENGTH) {
       throw valueFormatNotAllowedException(
           columnName,
-          output,
           "OBJECT",
           String.format(
-              "Object too large. length=%d maxLength=%d",
-              stringLength, MAX_SEMI_STRUCTURED_LENGTH));
+              "Object too large. length=%d maxLength=%d", stringLength, MAX_SEMI_STRUCTURED_LENGTH),
+          insertRowIndex);
     }
     return output;
   }
@@ -316,7 +322,11 @@ static String validateAndParseObject(String columnName, Object input) {
    * timestamps.
    */
   private static OffsetDateTime inputToOffsetDateTime(
-      String columnName, String typeName, Object input, ZoneId defaultTimezone) {
+      String columnName,
+      String typeName,
+      Object input,
+      ZoneId defaultTimezone,
+      final long insertRowIndex) {
     if (input instanceof OffsetDateTime) {
       return (OffsetDateTime) input;
     }
@@ -384,11 +394,11 @@ private static OffsetDateTime inputToOffsetDateTime(
       // Couldn't parse anything, throw an exception
       throw valueFormatNotAllowedException(
           columnName,
-          input.toString(),
           typeName,
           "Not a valid value, see"
               + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview"
-              + " for the list of supported formats");
+              + " for the list of supported formats",
+          insertRowIndex);
     }
 
     // Type is not supported, throw an exception
@@ -396,7 +406,8 @@ private static OffsetDateTime inputToOffsetDateTime(
         columnName,
         input.getClass(),
         typeName,
-        new String[] {"String", "LocalDate", "LocalDateTime", "ZonedDateTime", "OffsetDateTime"});
+        new String[] {"String", "LocalDate", "LocalDateTime", "ZonedDateTime", "OffsetDateTime"},
+        insertRowIndex);
   }
 
   private static <T> T catchParsingError(Supplier<T> op) {
@@ -427,12 +438,18 @@ private static <T> T catchParsingError(Supplier<T> op) {
    *     interpreted in the default timezone.
    * @param trimTimezone Whether timezone information should be removed from the resulting date,
    *     should be true for TIMESTAMP_NTZ columns.
+   * @param insertRowIndex
    * @return TimestampWrapper
    */
   static TimestampWrapper validateAndParseTimestamp(
-      String columnName, Object input, int scale, ZoneId defaultTimezone, boolean trimTimezone) {
+      String columnName,
+      Object input,
+      int scale,
+      ZoneId defaultTimezone,
+      boolean trimTimezone,
+      long insertRowIndex) {
     OffsetDateTime offsetDateTime =
-        inputToOffsetDateTime(columnName, "TIMESTAMP", input, defaultTimezone);
+        inputToOffsetDateTime(columnName, "TIMESTAMP", input, defaultTimezone, insertRowIndex);
 
     if (trimTimezone) {
       offsetDateTime = offsetDateTime.withOffsetSameLocal(ZoneOffset.UTC);
@@ -455,13 +472,14 @@ static TimestampWrapper validateAndParseTimestamp(
    * @param maxLengthOptional Maximum allowed length of the output String, if empty then uses
    *     maximum allowed by Snowflake
    *     (https://docs.snowflake.com/en/sql-reference/data-types-text.html#varchar)
+   * @param insertRowIndex
    */
   static String validateAndParseString(
-      String columnName, Object input, Optional<Integer> maxLengthOptional) {
+      String columnName, Object input, Optional<Integer> maxLengthOptional, long insertRowIndex) {
     String output;
     if (input instanceof String) {
       output = (String) input;
-      verifyValidUtf8(output, columnName, "STRING");
+      verifyValidUtf8(output, columnName, "STRING", insertRowIndex);
     } else if (input instanceof Number) {
       output = new BigDecimal(input.toString()).stripTrailingZeros().toPlainString();
     } else if (input instanceof Boolean || input instanceof Character) {
@@ -471,7 +489,8 @@ static String validateAndParseString(
           columnName,
           input.getClass(),
           "STRING",
-          new String[] {"String", "Number", "boolean", "char"});
+          new String[] {"String", "Number", "boolean", "char"},
+          insertRowIndex);
     }
     byte[] utf8Bytes = output.getBytes(StandardCharsets.UTF_8);
 
@@ -479,11 +498,10 @@ static String validateAndParseString(
     if (utf8Bytes.length > BYTES_16_MB) {
       throw valueFormatNotAllowedException(
           columnName,
-          input,
           "STRING",
           String.format(
-              "String too long: length=%d bytes maxLength=%d bytes",
-              utf8Bytes.length, BYTES_16_MB));
+              "String too long: length=%d bytes maxLength=%d bytes", utf8Bytes.length, BYTES_16_MB),
+          insertRowIndex);
     }
 
     // If max allowed length is specified (e.g. VARCHAR(10)), the number of unicode characters must
@@ -494,11 +512,11 @@ static String validateAndParseString(
           if (actualCharacters > maxAllowedCharacters) {
             throw valueFormatNotAllowedException(
                 columnName,
-                input,
                 "STRING",
                 String.format(
                     "String too long: length=%d characters maxLength=%d characters",
-                    actualCharacters, maxAllowedCharacters));
+                    actualCharacters, maxAllowedCharacters),
+                insertRowIndex);
           }
         });
     return output;
@@ -513,7 +531,8 @@ static String validateAndParseString(
    * <li>BigInteger, BigDecimal
    * <li>String
    */
-  static BigDecimal validateAndParseBigDecimal(String columnName, Object input) {
+  static BigDecimal validateAndParseBigDecimal(
+      String columnName, Object input, long insertRowIndex) {
     if (input instanceof BigDecimal) {
       return (BigDecimal) input;
     } else if (input instanceof BigInteger) {
@@ -530,7 +549,8 @@ static BigDecimal validateAndParseBigDecimal(String columnName, Object input) {
         final String stringInput = ((String) input).trim();
         return new BigDecimal(stringInput);
       } catch (NumberFormatException e) {
-        throw valueFormatNotAllowedException(columnName, input, "NUMBER", "Not a valid number");
+        throw valueFormatNotAllowedException(
+            columnName, "NUMBER", "Not a valid number", insertRowIndex);
       }
     } else {
       throw typeNotAllowedException(
@@ -539,7 +559,8 @@ static BigDecimal validateAndParseBigDecimal(String columnName, Object input) {
           "NUMBER",
           new String[] {
             "int", "long", "byte", "short", "float", "double", "BigDecimal", "BigInteger", "String"
-          });
+          },
+          insertRowIndex);
     }
   }
 
@@ -555,9 +576,9 @@ static BigDecimal validateAndParseBigDecimal(String columnName, Object input) {
    *   <li>{@link Instant}
    * </ul>
    */
-  static int validateAndParseDate(String columnName, Object input) {
+  static int validateAndParseDate(String columnName, Object input, long insertRowIndex) {
     OffsetDateTime offsetDateTime =
-        inputToOffsetDateTime(columnName, "DATE", input, ZoneOffset.UTC);
+        inputToOffsetDateTime(columnName, "DATE", input, ZoneOffset.UTC, insertRowIndex);
     return Math.toIntExact(offsetDateTime.toLocalDate().toEpochDay());
   }
 
@@ -572,10 +593,11 @@ static int validateAndParseDate(String columnName, Object input) {
    * @param input Array to validate
    * @param maxLengthOptional Max array length, defaults to 8MB, which is the max allowed length for
    *     BINARY column
+   * @param insertRowIndex
    * @return Validated array
    */
   static byte[] validateAndParseBinary(
-      String columnName, Object input, Optional<Integer> maxLengthOptional) {
+      String columnName, Object input, Optional<Integer> maxLengthOptional, long insertRowIndex) {
     byte[] output;
     if (input instanceof byte[]) {
       // byte[] is a mutable object, we need to create a defensive copy to protect against
@@ -589,20 +611,25 @@ static byte[] validateAndParseBinary(
         String stringInput = ((String) input).trim();
         output = Hex.decodeHex(stringInput);
       } catch (DecoderException e) {
-        throw valueFormatNotAllowedException(columnName, input, "BINARY", "Not a valid hex string");
+        throw valueFormatNotAllowedException(
+            columnName, "BINARY", "Not a valid hex string", insertRowIndex);
       }
     } else {
       throw typeNotAllowedException(
-          columnName, input.getClass(), "BINARY", new String[] {"byte[]", "String"});
+          columnName,
+          input.getClass(),
+          "BINARY",
+          new String[] {"byte[]", "String"},
+          insertRowIndex);
     }
 
     int maxLength = maxLengthOptional.orElse(BYTES_8_MB);
     if (output.length > maxLength) {
       throw valueFormatNotAllowedException(
           columnName,
-          String.format("byte[%d]", output.length),
           "BINARY",
-          String.format("Binary too long: length=%d maxLength=%d", output.length, maxLength));
+          String.format("Binary too long: length=%d maxLength=%d", output.length, maxLength),
+          insertRowIndex);
     }
     return output;
   }
@@ -617,19 +644,21 @@ static byte[] validateAndParseBinary(
    *   <li>{@link OffsetTime}
    * </ul>
    */
-  static BigInteger validateAndParseTime(String columnName, Object input, int scale) {
+  static BigInteger validateAndParseTime(
+      String columnName, Object input, int scale, long insertRowIndex) {
     if (input instanceof LocalTime) {
       LocalTime localTime = (LocalTime) input;
       return BigInteger.valueOf(localTime.toNanoOfDay()).divide(Power10.sb16Table[9 - scale]);
     } else if (input instanceof OffsetTime) {
-      return validateAndParseTime(columnName, ((OffsetTime) input).toLocalTime(), scale);
+      return validateAndParseTime(
+          columnName, ((OffsetTime) input).toLocalTime(), scale, insertRowIndex);
     } else if (input instanceof String) {
       String stringInput = ((String) input).trim();
       {
         // First, try to parse LocalTime
         LocalTime localTime = catchParsingError(() -> LocalTime.parse(stringInput));
         if (localTime != null) {
-          return validateAndParseTime(columnName, localTime, scale);
+          return validateAndParseTime(columnName, localTime, scale, insertRowIndex);
         }
       }
 
@@ -637,7 +666,7 @@ static BigInteger validateAndParseTime(String columnName, Object input, int scal
         // Alternatively, try to parse OffsetTime
         OffsetTime offsetTime = catchParsingError((() -> OffsetTime.parse(stringInput)));
         if (offsetTime != null) {
-          return validateAndParseTime(columnName, offsetTime.toLocalTime(), scale);
+          return validateAndParseTime(columnName, offsetTime.toLocalTime(), scale, insertRowIndex);
         }
       }
 
@@ -648,21 +677,26 @@ static BigInteger validateAndParseTime(String columnName, Object input, int scal
           return validateAndParseTime(
               columnName,
               LocalDateTime.ofInstant(parsedInstant, ZoneOffset.UTC).toLocalTime(),
-              scale);
+              scale,
+              insertRowIndex);
         }
       }
 
       throw valueFormatNotAllowedException(
           columnName,
-          input,
           "TIME",
           "Not a valid time, see"
               + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview"
-              + " for the list of supported formats");
+              + " for the list of supported formats",
+          insertRowIndex);
 
     } else {
       throw typeNotAllowedException(
-          columnName, input.getClass(), "TIME", new String[] {"String", "LocalTime", "OffsetTime"});
+          columnName,
+          input.getClass(),
+          "TIME",
+          new String[] {"String", "LocalTime", "OffsetTime"},
+          insertRowIndex);
     }
   }
 
@@ -701,8 +735,9 @@ private static Instant parseInstantGuessScale(String input) {
    * </ul>
    *
    * @param input
+   * @param insertRowIndex
    */
-  static double validateAndParseReal(String columnName, Object input) {
+  static double validateAndParseReal(String columnName, Object input, long insertRowIndex) {
     if (input instanceof Float) {
       return Double.parseDouble(input.toString());
     } else if (input instanceof Number) {
@@ -722,12 +757,12 @@ static double validateAndParseReal(String columnName, Object input) {
             return Double.NEGATIVE_INFINITY;
           default:
             throw valueFormatNotAllowedException(
-                columnName, input, "REAL", "Not a valid decimal number");
+                columnName, "REAL", "Not a valid decimal number", insertRowIndex);
         }
       }
     }
     throw typeNotAllowedException(
-        columnName, input.getClass(), "REAL", new String[] {"Number", "String"});
+        columnName, input.getClass(), "REAL", new String[] {"Number", "String"}, insertRowIndex);
   }
 
   /**
@@ -744,43 +779,48 @@ static double validateAndParseReal(String columnName, Object input) {
    * @param input Input to be converted
    * @return 1 or 0 where 1=true, 0=false
    */
-  static int validateAndParseBoolean(String columnName, Object input) {
+  static int validateAndParseBoolean(String columnName, Object input, long insertRowIndex) {
     if (input instanceof Boolean) {
       return (boolean) input ? 1 : 0;
     } else if (input instanceof Number) {
       return new BigDecimal(input.toString()).compareTo(BigDecimal.ZERO) == 0 ? 0 : 1;
     } else if (input instanceof String) {
-      return convertStringToBoolean(columnName, (String) input) ? 1 : 0;
+      return convertStringToBoolean(columnName, (String) input, insertRowIndex) ? 1 : 0;
     }
 
     throw typeNotAllowedException(
-        columnName, input.getClass(), "BOOLEAN", new String[] {"boolean", "Number", "String"});
+        columnName,
+        input.getClass(),
+        "BOOLEAN",
+        new String[] {"boolean", "Number", "String"},
+        insertRowIndex);
   }
 
-  static void checkValueInRange(BigDecimal bigDecimalValue, int scale, int precision) {
+  static void checkValueInRange(
+      BigDecimal bigDecimalValue, int scale, int precision, final long insertRowIndex) {
     if (bigDecimalValue.abs().compareTo(BigDecimal.TEN.pow(precision - scale)) >= 0) {
       throw new SFException(
-          ErrorCode.INVALID_ROW,
-          bigDecimalValue,
+          ErrorCode.INVALID_FORMAT_ROW,
           String.format(
-              "Number out of representable exclusive range of (-1e%s..1e%s)",
-              precision - scale, precision - scale));
+              "Number out of representable exclusive range of (-1e%s..1e%s), Row Index:%s",
+              precision - scale, precision - scale, insertRowIndex));
     }
   }
 
   static Set<String> allowedBooleanStringsLowerCased =
       Sets.newHashSet("1", "0", "yes", "no", "y", "n", "t", "f", "true", "false", "on", "off");
 
-  private static boolean convertStringToBoolean(String columnName, String value) {
+  private static boolean convertStringToBoolean(
+      String columnName, String value, final long insertRowIndex) {
     String normalizedInput = value.toLowerCase().trim();
     if (!allowedBooleanStringsLowerCased.contains(normalizedInput)) {
       throw valueFormatNotAllowedException(
           columnName,
-          value,
           "BOOLEAN",
           "Not a valid boolean, see"
               + " https://docs.snowflake.com/en/sql-reference/data-types-logical.html#conversion-to-boolean"
-              + " for the list of supported formats");
+              + " for the list of supported formats",
+          insertRowIndex);
     }
     return "1".equals(normalizedInput)
         || "yes".equals(normalizedInput)
@@ -798,12 +838,17 @@ private static boolean convertStringToBoolean(String columnName, String value) {
    * @param allowedJavaTypes Java types supported for the Java type
    */
   private static SFException typeNotAllowedException(
-      String columnName, Class<?> javaType, String snowflakeType, String[] allowedJavaTypes) {
+      String columnName,
+      Class<?> javaType,
+      String snowflakeType,
+      String[] allowedJavaTypes,
+      final long insertRowIndex) {
     return new SFException(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         String.format(
-            "Object of type %s cannot be ingested into Snowflake column %s of type %s",
-            javaType.getName(), columnName, snowflakeType),
+            "Object of type %s cannot be ingested into Snowflake column %s of type %s, Row"
+                + " Index:%s",
+            javaType.getName(), columnName, snowflakeType, insertRowIndex),
         String.format(
             String.format("Allowed Java types: %s", String.join(", ", allowedJavaTypes))));
   }
@@ -812,40 +857,35 @@ private static SFException typeNotAllowedException(
    * Create exception when the Java type is correct, but the value is invalid (e.g. boolean cannot
    * be parsed from a string)
    *
-   * @param value Invalid value causing the exception
+   * <p>Note: Do not log actual Object Value
+   *
+   * @param columnName Column Name
    * @param snowflakeType Snowflake column type
+   * @param reason Reason why value format is not allowed.
+   * @param rowIndex Index of the Input row primarily for debugging purposes.
+   * @return SFException is thrown
    */
   private static SFException valueFormatNotAllowedException(
-      String columnName, Object value, String snowflakeType, String reason) {
+      String columnName, String snowflakeType, String reason, final long rowIndex) {
     return new SFException(
-        ErrorCode.INVALID_ROW,
-        sanitizeValueForExceptionMessage(value),
+        ErrorCode.INVALID_VALUE_ROW,
         String.format(
-            "Value cannot be ingested into Snowflake column %s of type %s: %s",
-            columnName, snowflakeType, reason));
-  }
-
-  /**
-   * Before passing a value to an exception string, we should limit how many characters are
-   * displayed. Important especially for "value exceeds max column size" exceptions.
-   *
-   * @param value Value to adapt for exception message
-   */
-  private static String sanitizeValueForExceptionMessage(Object value) {
-    int maxSize = 20;
-    String valueString = value.toString();
-    return valueString.length() <= maxSize ? valueString : valueString.substring(0, 20) + "...";
+            "Value cannot be ingested into Snowflake column %s of type %s, Row Index: %s, reason:"
+                + " %s",
+            columnName, snowflakeType, rowIndex, reason));
   }
 
   /**
    * Validates that a string is valid UTF-8 string. It catches situations like unmatched high/low
    * UTF-16 surrogate, for example.
    */
-  private static void verifyValidUtf8(String input, String columnName, String dataType) {
+  private static void verifyValidUtf8(
+      String input, String columnName, String dataType, final long insertRowIndex) {
     String roundTripStr =
         new String(input.getBytes(StandardCharsets.UTF_8), StandardCharsets.UTF_8);
     if (!input.equals(roundTripStr)) {
-      throw valueFormatNotAllowedException(columnName, input, dataType, "Invalid Unicode string");
+      throw valueFormatNotAllowedException(
+          columnName, dataType, "Invalid Unicode string", insertRowIndex);
     }
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 9a37831e8..1425a5e9c 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -140,10 +140,12 @@ boolean hasColumn(String name) {
   @Override
   float addRow(
       Map<String, Object> row,
-      int curRowIndex,
+      int bufferedRowIndex,
       Map<String, RowBufferStats> statsMap,
-      Set<String> formattedInputColumnNames) {
-    return addRow(row, this::writeRow, statsMap, formattedInputColumnNames);
+      Set<String> formattedInputColumnNames,
+      final long insertRowIndex) {
+    return addRow(
+        row, bufferedRowIndex, this::writeRow, statsMap, formattedInputColumnNames, insertRowIndex);
   }
 
   void writeRow(List<Object> row) {
@@ -159,14 +161,17 @@ float addTempRow(
       Map<String, Object> row,
       int curRowIndex,
       Map<String, RowBufferStats> statsMap,
-      Set<String> formattedInputColumnNames) {
-    return addRow(row, this::writeRow, statsMap, formattedInputColumnNames);
+      Set<String> formattedInputColumnNames,
+      long insertRowIndex) {
+    return addRow(
+        row, curRowIndex, this::writeRow, statsMap, formattedInputColumnNames, insertRowIndex);
   }
 
   /**
    * Adds a row to the parquet buffer.
    *
    * @param row row to add
+   * @param curRowIndex current row index
    * @param out internal buffer to add to
    * @param statsMap column stats map
    * @param inputColumnNames list of input column names after formatting
@@ -174,9 +179,11 @@ float addTempRow(
    */
   private float addRow(
       Map<String, Object> row,
+      int curRowIndex,
       Consumer<List<Object>> out,
       Map<String, RowBufferStats> statsMap,
-      Set<String> inputColumnNames) {
+      Set<String> inputColumnNames,
+      final long insertRowIndex) {
     Object[] indexedRow = new Object[fieldIndex.size()];
     float size = 0F;
 
@@ -196,7 +203,7 @@ private float addRow(
           columnDescriptor.getPrimitiveType().getPrimitiveTypeName();
       ParquetValueParser.ParquetBufferValue valueWithSize =
           ParquetValueParser.parseColumnValueToParquet(
-              value, column, typeName, forkedStats, defaultTimezone);
+              value, column, typeName, forkedStats, defaultTimezone, curRowIndex);
       indexedRow[colIndex] = valueWithSize.getValue();
       size += valueWithSize.getSize();
     }
@@ -244,7 +251,7 @@ Optional<ParquetChunkData> getSnapshot(final String filePath) {
     if (!enableParquetInternalBuffering) {
       data.forEach(r -> oldData.add(new ArrayList<>(r)));
     }
-    return rowCount <= 0
+    return bufferedRowCount <= 0
         ? Optional.empty()
         : Optional.of(new ParquetChunkData(oldData, bdecParquetWriter, fileOutput, metadata));
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
index 8d1f131fc..a5785c85d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
@@ -75,6 +75,7 @@ float getSize() {
    * @param columnMetadata column metadata
    * @param typeName Parquet primitive type name
    * @param stats column stats to update
+   * @param curRowIndex Row index corresponding the row to parse
    * @return parsed value and byte size of Parquet internal representation
    */
   static ParquetBufferValue parseColumnValueToParquet(
@@ -82,7 +83,8 @@ static ParquetBufferValue parseColumnValueToParquet(
       ColumnMetadata columnMetadata,
       PrimitiveType.PrimitiveTypeName typeName,
       RowBufferStats stats,
-      ZoneId defaultTimezone) {
+      ZoneId defaultTimezone,
+      int curRowIndex) {
     Utils.assertNotNull("Parquet column stats", stats);
     float estimatedParquetSize = 0F;
     estimatedParquetSize += DEFINITION_LEVEL_ENCODING_BYTE_LEN;
@@ -94,7 +96,8 @@ static ParquetBufferValue parseColumnValueToParquet(
       switch (typeName) {
         case BOOLEAN:
           int intValue =
-              DataValidationUtil.validateAndParseBoolean(columnMetadata.getName(), value);
+              DataValidationUtil.validateAndParseBoolean(
+                  columnMetadata.getName(), value, curRowIndex);
           value = intValue > 0;
           stats.addIntValue(BigInteger.valueOf(intValue));
           estimatedParquetSize += BIT_ENCODING_BYTE_LEN;
@@ -107,7 +110,8 @@ static ParquetBufferValue parseColumnValueToParquet(
                   columnMetadata.getScale(),
                   Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
                   logicalType,
-                  physicalType);
+                  physicalType,
+                  curRowIndex);
           value = intVal;
           stats.addIntValue(BigInteger.valueOf(intVal));
           estimatedParquetSize += 4;
@@ -121,14 +125,15 @@ static ParquetBufferValue parseColumnValueToParquet(
                   Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
                   logicalType,
                   physicalType,
-                  defaultTimezone);
+                  defaultTimezone,
+                  curRowIndex);
           value = longValue;
           stats.addIntValue(BigInteger.valueOf(longValue));
           estimatedParquetSize += 8;
           break;
         case DOUBLE:
           double doubleValue =
-              DataValidationUtil.validateAndParseReal(columnMetadata.getName(), value);
+              DataValidationUtil.validateAndParseReal(columnMetadata.getName(), value, curRowIndex);
           value = doubleValue;
           stats.addRealValue(doubleValue);
           estimatedParquetSize += 8;
@@ -136,10 +141,10 @@ static ParquetBufferValue parseColumnValueToParquet(
         case BINARY:
           int length = 0;
           if (logicalType == AbstractRowBuffer.ColumnLogicalType.BINARY) {
-            value = getBinaryValueForLogicalBinary(value, stats, columnMetadata);
+            value = getBinaryValueForLogicalBinary(value, stats, columnMetadata, curRowIndex);
             length = ((byte[]) value).length;
           } else {
-            String str = getBinaryValue(value, stats, columnMetadata);
+            String str = getBinaryValue(value, stats, columnMetadata, curRowIndex);
             value = str;
             if (str != null) {
               length = str.getBytes().length;
@@ -159,7 +164,8 @@ static ParquetBufferValue parseColumnValueToParquet(
                   Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
                   logicalType,
                   physicalType,
-                  defaultTimezone);
+                  defaultTimezone,
+                  curRowIndex);
           stats.addIntValue(intRep);
           value = getSb16Bytes(intRep);
           estimatedParquetSize += 16;
@@ -172,7 +178,9 @@ static ParquetBufferValue parseColumnValueToParquet(
     if (value == null) {
       if (!columnMetadata.getNullable()) {
         throw new SFException(
-            ErrorCode.INVALID_ROW, columnMetadata.getName(), "Passed null to non nullable field");
+            ErrorCode.INVALID_FORMAT_ROW,
+            columnMetadata.getName(),
+            "Passed null to non nullable field");
       }
       stats.incCurrentNullCount();
     }
@@ -196,21 +204,24 @@ private static int getInt32Value(
       @Nullable Integer scale,
       Integer precision,
       AbstractRowBuffer.ColumnLogicalType logicalType,
-      AbstractRowBuffer.ColumnPhysicalType physicalType) {
+      AbstractRowBuffer.ColumnPhysicalType physicalType,
+      final int curRowIndex) {
     int intVal;
     switch (logicalType) {
       case DATE:
-        intVal = DataValidationUtil.validateAndParseDate(columnName, value);
+        intVal = DataValidationUtil.validateAndParseDate(columnName, value, curRowIndex);
         break;
       case TIME:
         Utils.assertNotNull("Unexpected null scale for TIME data type", scale);
-        intVal = DataValidationUtil.validateAndParseTime(columnName, value, scale).intValue();
+        intVal =
+            DataValidationUtil.validateAndParseTime(columnName, value, scale, curRowIndex)
+                .intValue();
         break;
       case FIXED:
         BigDecimal bigDecimalValue =
-            DataValidationUtil.validateAndParseBigDecimal(columnName, value);
+            DataValidationUtil.validateAndParseBigDecimal(columnName, value, curRowIndex);
         bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
-        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
+        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision, curRowIndex);
         intVal = bigDecimalValue.intValue();
         break;
       default:
@@ -236,34 +247,37 @@ private static long getInt64Value(
       int precision,
       AbstractRowBuffer.ColumnLogicalType logicalType,
       AbstractRowBuffer.ColumnPhysicalType physicalType,
-      ZoneId defaultTimezone) {
+      ZoneId defaultTimezone,
+      final int curRowIndex) {
     long longValue;
     switch (logicalType) {
       case TIME:
         Utils.assertNotNull("Unexpected null scale for TIME data type", scale);
-        longValue = DataValidationUtil.validateAndParseTime(columnName, value, scale).longValue();
+        longValue =
+            DataValidationUtil.validateAndParseTime(columnName, value, scale, curRowIndex)
+                .longValue();
         break;
       case TIMESTAMP_LTZ:
       case TIMESTAMP_NTZ:
         boolean trimTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
         longValue =
             DataValidationUtil.validateAndParseTimestamp(
-                    columnName, value, scale, defaultTimezone, trimTimezone)
+                    columnName, value, scale, defaultTimezone, trimTimezone, curRowIndex)
                 .toBinary(false)
                 .longValue();
         break;
       case TIMESTAMP_TZ:
         longValue =
             DataValidationUtil.validateAndParseTimestamp(
-                    columnName, value, scale, defaultTimezone, false)
+                    columnName, value, scale, defaultTimezone, false, curRowIndex)
                 .toBinary(true)
                 .longValue();
         break;
       case FIXED:
         BigDecimal bigDecimalValue =
-            DataValidationUtil.validateAndParseBigDecimal(columnName, value);
+            DataValidationUtil.validateAndParseBigDecimal(columnName, value, curRowIndex);
         bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
-        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
+        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision, curRowIndex);
         longValue = bigDecimalValue.longValue();
         break;
       default:
@@ -289,24 +303,25 @@ private static BigInteger getSb16Value(
       int precision,
       AbstractRowBuffer.ColumnLogicalType logicalType,
       AbstractRowBuffer.ColumnPhysicalType physicalType,
-      ZoneId defaultTimezone) {
+      ZoneId defaultTimezone,
+      final int curRowIndex) {
     switch (logicalType) {
       case TIMESTAMP_TZ:
         return DataValidationUtil.validateAndParseTimestamp(
-                columnName, value, scale, defaultTimezone, false)
+                columnName, value, scale, defaultTimezone, false, curRowIndex)
             .toBinary(true);
       case TIMESTAMP_LTZ:
       case TIMESTAMP_NTZ:
         boolean trimTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
         return DataValidationUtil.validateAndParseTimestamp(
-                columnName, value, scale, defaultTimezone, trimTimezone)
+                columnName, value, scale, defaultTimezone, trimTimezone, curRowIndex)
             .toBinary(false);
       case FIXED:
         BigDecimal bigDecimalValue =
-            DataValidationUtil.validateAndParseBigDecimal(columnName, value);
+            DataValidationUtil.validateAndParseBigDecimal(columnName, value, curRowIndex);
         // explicitly match the BigDecimal input scale with the Snowflake data type scale
         bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
-        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision);
+        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision, curRowIndex);
         return bigDecimalValue.unscaledValue();
       default:
         throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
@@ -339,20 +354,26 @@ static byte[] getSb16Bytes(BigInteger intRep) {
    * @return string representation
    */
   private static String getBinaryValue(
-      Object value, RowBufferStats stats, ColumnMetadata columnMetadata) {
+      Object value, RowBufferStats stats, ColumnMetadata columnMetadata, final int curRowIndex) {
     AbstractRowBuffer.ColumnLogicalType logicalType =
         AbstractRowBuffer.ColumnLogicalType.valueOf(columnMetadata.getLogicalType());
     String str;
     if (logicalType.isObject()) {
       switch (logicalType) {
         case OBJECT:
-          str = DataValidationUtil.validateAndParseObject(columnMetadata.getName(), value);
+          str =
+              DataValidationUtil.validateAndParseObject(
+                  columnMetadata.getName(), value, curRowIndex);
           break;
         case VARIANT:
-          str = DataValidationUtil.validateAndParseVariant(columnMetadata.getName(), value);
+          str =
+              DataValidationUtil.validateAndParseVariant(
+                  columnMetadata.getName(), value, curRowIndex);
           break;
         case ARRAY:
-          str = DataValidationUtil.validateAndParseArray(columnMetadata.getName(), value);
+          str =
+              DataValidationUtil.validateAndParseArray(
+                  columnMetadata.getName(), value, curRowIndex);
           break;
         default:
           throw new SFException(
@@ -362,7 +383,10 @@ private static String getBinaryValue(
       String maxLengthString = columnMetadata.getLength().toString();
       str =
           DataValidationUtil.validateAndParseString(
-              columnMetadata.getName(), value, Optional.of(maxLengthString).map(Integer::parseInt));
+              columnMetadata.getName(),
+              value,
+              Optional.of(maxLengthString).map(Integer::parseInt),
+              curRowIndex);
       stats.addStrValue(str);
     }
     return str;
@@ -376,11 +400,14 @@ private static String getBinaryValue(
    * @return byte array representation
    */
   private static byte[] getBinaryValueForLogicalBinary(
-      Object value, RowBufferStats stats, ColumnMetadata columnMetadata) {
+      Object value, RowBufferStats stats, ColumnMetadata columnMetadata, final int curRowIndex) {
     String maxLengthString = columnMetadata.getByteLength().toString();
     byte[] bytes =
         DataValidationUtil.validateAndParseBinary(
-            columnMetadata.getName(), value, Optional.of(maxLengthString).map(Integer::parseInt));
+            columnMetadata.getName(),
+            value,
+            Optional.of(maxLengthString).map(Integer::parseInt),
+            curRowIndex);
     stats.addBinaryValue(bytes);
     return bytes;
   }
diff --git a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
index 3b3c4f9f7..bacb3d556 100644
--- a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
+++ b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
@@ -9,7 +9,7 @@ public enum ErrorCode {
   INTERNAL_ERROR("0001"),
   NULL_VALUE("0002"),
   NULL_OR_EMPTY_STRING("0003"),
-  INVALID_ROW("0004"),
+  INVALID_FORMAT_ROW("0004"),
   UNKNOWN_DATA_TYPE("0005"),
   REGISTER_BLOB_FAILURE("0006"),
   OPEN_CHANNEL_FAILURE("0007"),
@@ -34,7 +34,8 @@ public enum ErrorCode {
   INVALID_COLLATION_STRING("0026"),
   ENCRYPTION_FAILURE("0027"),
   CHANNEL_STATUS_INVALID("0028"),
-  UNSUPPORTED_DATA_TYPE("0029");
+  UNSUPPORTED_DATA_TYPE("0029"),
+  INVALID_VALUE_ROW("0030");
 
   public static final String errorMessageResource = "net.snowflake.ingest.ingest_error_messages";
 
diff --git a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
index 49ac31d92..1008bcb8b 100644
--- a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
+++ b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
@@ -32,3 +32,4 @@
 0027=Failure during data encryption.
 0028=Get channel status indicates Channel {0} is invalid with status code {1}, please reopening the channel.
 0029=Data type not supported: {0}
+0030=The given row cannot be converted to the internal format due to invalid value: {0}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 9c3279c41..6f3dd4558 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -72,451 +72,515 @@ private void expectError(ErrorCode expectedErrorCode, Runnable action) {
 
   @Test
   public void testValidateAndParseDate() {
-    assertEquals(9, validateAndParseDate("COL", LocalDate.of(1970, 1, 10)));
-    assertEquals(9, validateAndParseDate("COL", LocalDateTime.of(1970, 1, 10, 1, 0)));
+    assertEquals(9, validateAndParseDate("COL", LocalDate.of(1970, 1, 10), 0));
+    assertEquals(9, validateAndParseDate("COL", LocalDateTime.of(1970, 1, 10, 1, 0), 0));
     assertEquals(
         9,
         validateAndParseDate(
-            "COL", OffsetDateTime.of(1970, 1, 10, 1, 0, 34, 123456789, ZoneOffset.of("-07:00"))));
+            "COL",
+            OffsetDateTime.of(1970, 1, 10, 1, 0, 34, 123456789, ZoneOffset.of("-07:00")),
+            0));
     assertEquals(
         9,
         validateAndParseDate(
-            "COL", OffsetDateTime.of(1970, 1, 10, 1, 0, 34, 123456789, ZoneOffset.of("+07:00"))));
+            "COL",
+            OffsetDateTime.of(1970, 1, 10, 1, 0, 34, 123456789, ZoneOffset.of("+07:00")),
+            0));
     assertEquals(
         9,
         validateAndParseDate(
             "COL",
-            ZonedDateTime.of(1970, 1, 10, 1, 0, 34, 123456789, ZoneId.of("America/Los_Angeles"))));
+            ZonedDateTime.of(1970, 1, 10, 1, 0, 34, 123456789, ZoneId.of("America/Los_Angeles")),
+            0));
     assertEquals(
         9,
         validateAndParseDate(
-            "COL", ZonedDateTime.of(1970, 1, 10, 1, 0, 34, 123456789, ZoneId.of("Asia/Tokyo"))));
-    assertEquals(19380, validateAndParseDate("COL", Instant.ofEpochMilli(1674478926000L)));
-
-    assertEquals(-923, validateAndParseDate("COL", "1967-06-23"));
-    assertEquals(-923, validateAndParseDate("COL", "  1967-06-23 \t\n"));
-    assertEquals(-923, validateAndParseDate("COL", "1967-06-23T01:01:01"));
-    assertEquals(18464, validateAndParseDate("COL", "2020-07-21"));
-    assertEquals(18464, validateAndParseDate("COL", "2020-07-21T23:31:00"));
-    assertEquals(18464, validateAndParseDate("COL", "2020-07-21T23:31:00+07:00"));
-    assertEquals(18464, validateAndParseDate("COL", "2020-07-21T23:31:00-07:00"));
+            "COL", ZonedDateTime.of(1970, 1, 10, 1, 0, 34, 123456789, ZoneId.of("Asia/Tokyo")), 0));
+    assertEquals(19380, validateAndParseDate("COL", Instant.ofEpochMilli(1674478926000L), 0));
+
+    assertEquals(-923, validateAndParseDate("COL", "1967-06-23", 0));
+    assertEquals(-923, validateAndParseDate("COL", "  1967-06-23 \t\n", 0));
+    assertEquals(-923, validateAndParseDate("COL", "1967-06-23T01:01:01", 0));
+    assertEquals(18464, validateAndParseDate("COL", "2020-07-21", 0));
+    assertEquals(18464, validateAndParseDate("COL", "2020-07-21T23:31:00", 0));
+    assertEquals(18464, validateAndParseDate("COL", "2020-07-21T23:31:00+07:00", 0));
+    assertEquals(18464, validateAndParseDate("COL", "2020-07-21T23:31:00-07:00", 0));
     assertEquals(
-        18464, validateAndParseDate("COL", "2020-07-21T23:31:00-07:00[America/Los_Angeles]"));
-    assertEquals(18464, validateAndParseDate("COL", "2020-07-21T23:31:00+09:00[Asia/Tokyo]"));
+        18464, validateAndParseDate("COL", "2020-07-21T23:31:00-07:00[America/Los_Angeles]", 0));
+    assertEquals(18464, validateAndParseDate("COL", "2020-07-21T23:31:00+09:00[Asia/Tokyo]", 0));
 
     // Test integer-stored date
-    assertEquals(19380, validateAndParseDate("COL", "1674478926"));
-    assertEquals(19380, validateAndParseDate("COL", "1674478926000"));
-    assertEquals(19380, validateAndParseDate("COL", "1674478926000000"));
-    assertEquals(19380, validateAndParseDate("COL", "1674478926000000000"));
+    assertEquals(19380, validateAndParseDate("COL", "1674478926", 0));
+    assertEquals(19380, validateAndParseDate("COL", "1674478926000", 0));
+    assertEquals(19380, validateAndParseDate("COL", "1674478926000000", 0));
+    assertEquals(19380, validateAndParseDate("COL", "1674478926000000000", 0));
 
     // Time input is not supported
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "20:57:01"));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseDate("COL", "20:57:01", 0));
 
     // Test forbidden values
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", new Object()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", LocalTime.now()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", OffsetTime.now()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", new java.util.Date()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", ""));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "foo"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", "1.0"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", 'c'));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", 1));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", 1L));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", 1.25));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", BigInteger.valueOf(1)));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseDate("COL", BigDecimal.valueOf(1.25)));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseDate("COL", new Object(), 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseDate("COL", LocalTime.now(), 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseDate("COL", OffsetTime.now(), 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseDate("COL", new java.util.Date(), 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseDate("COL", false, 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseDate("COL", "", 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseDate("COL", "foo", 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseDate("COL", "1.0", 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseDate("COL", 'c', 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseDate("COL", 1, 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseDate("COL", 1L, 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseDate("COL", 1.25, 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseDate("COL", BigInteger.valueOf(1), 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseDate("COL", BigDecimal.valueOf(1.25), 0));
   }
 
   @Test
   public void testValidateAndParseTime() {
     // Test local time
-    assertEquals(46920, validateAndParseTime("COL", "13:02", 0).longValueExact());
-    assertEquals(46920, validateAndParseTime("COL", "  13:02 \t\n", 0).longValueExact());
-    assertEquals(46926, validateAndParseTime("COL", "13:02:06", 0).longValueExact());
-    assertEquals(469260, validateAndParseTime("COL", "13:02:06", 1).longValueExact());
-    assertEquals(46926000000000L, validateAndParseTime("COL", "13:02:06", 9).longValueExact());
-
-    assertEquals(46926, validateAndParseTime("COL", "13:02:06.1234", 0).longValueExact());
-    assertEquals(469261, validateAndParseTime("COL", "13:02:06.1234", 1).longValueExact());
-    assertEquals(46926123400000L, validateAndParseTime("COL", "13:02:06.1234", 9).longValueExact());
-
-    assertEquals(46926, validateAndParseTime("COL", "13:02:06.123456789", 0).longValueExact());
-    assertEquals(469261, validateAndParseTime("COL", "13:02:06.123456789", 1).longValueExact());
+    assertEquals(46920, validateAndParseTime("COL", "13:02", 0, 0).longValueExact());
+    assertEquals(46920, validateAndParseTime("COL", "  13:02 \t\n", 0, 0).longValueExact());
+    assertEquals(46926, validateAndParseTime("COL", "13:02:06", 0, 0).longValueExact());
+    assertEquals(469260, validateAndParseTime("COL", "13:02:06", 1, 0).longValueExact());
+    assertEquals(46926000000000L, validateAndParseTime("COL", "13:02:06", 9, 0).longValueExact());
+
+    assertEquals(46926, validateAndParseTime("COL", "13:02:06.1234", 0, 0).longValueExact());
+    assertEquals(469261, validateAndParseTime("COL", "13:02:06.1234", 1, 0).longValueExact());
     assertEquals(
-        46926123456789L, validateAndParseTime("COL", "13:02:06.123456789", 9).longValueExact());
+        46926123400000L, validateAndParseTime("COL", "13:02:06.1234", 9, 0).longValueExact());
+
+    assertEquals(46926, validateAndParseTime("COL", "13:02:06.123456789", 0, 0).longValueExact());
+    assertEquals(469261, validateAndParseTime("COL", "13:02:06.123456789", 1, 0).longValueExact());
+    assertEquals(
+        46926123456789L, validateAndParseTime("COL", "13:02:06.123456789", 9, 0).longValueExact());
 
     // Test that offset time does not make any difference
     assertEquals(
         46926123456789L,
-        validateAndParseTime("COL", "13:02:06.123456789+09:00", 9).longValueExact());
+        validateAndParseTime("COL", "13:02:06.123456789+09:00", 9, 0).longValueExact());
     assertEquals(
         46926123456789L,
-        validateAndParseTime("COL", "13:02:06.123456789-09:00", 9).longValueExact());
+        validateAndParseTime("COL", "13:02:06.123456789-09:00", 9, 0).longValueExact());
 
     // Test integer-stored time and scale guessing
-    assertEquals(46926L, validateAndParseTime("COL", "1674478926", 0).longValueExact());
-    assertEquals(46926L, validateAndParseTime("COL", "1674478926123", 0).longValueExact());
-    assertEquals(46926L, validateAndParseTime("COL", "1674478926123456", 0).longValueExact());
-    assertEquals(46926L, validateAndParseTime("COL", "1674478926123456789", 0).longValueExact());
-
-    assertEquals(469260L, validateAndParseTime("COL", "1674478926", 1).longValueExact());
-    assertEquals(469261L, validateAndParseTime("COL", "1674478926123", 1).longValueExact());
-    assertEquals(469261L, validateAndParseTime("COL", "1674478926123456", 1).longValueExact());
-    assertEquals(469261L, validateAndParseTime("COL", "1674478926123456789", 1).longValueExact());
-
-    assertEquals(46926000000000L, validateAndParseTime("COL", "1674478926", 9).longValueExact());
-    assertEquals(46926123000000L, validateAndParseTime("COL", "1674478926123", 9).longValueExact());
+    assertEquals(46926L, validateAndParseTime("COL", "1674478926", 0, 0).longValueExact());
+    assertEquals(46926L, validateAndParseTime("COL", "1674478926123", 0, 0).longValueExact());
+    assertEquals(46926L, validateAndParseTime("COL", "1674478926123456", 0, 0).longValueExact());
+    assertEquals(46926L, validateAndParseTime("COL", "1674478926123456789", 0, 0).longValueExact());
+
+    assertEquals(469260L, validateAndParseTime("COL", "1674478926", 1, 0).longValueExact());
+    assertEquals(469261L, validateAndParseTime("COL", "1674478926123", 1, 0).longValueExact());
+    assertEquals(469261L, validateAndParseTime("COL", "1674478926123456", 1, 0).longValueExact());
+    assertEquals(
+        469261L, validateAndParseTime("COL", "1674478926123456789", 1, 0).longValueExact());
+
+    assertEquals(46926000000000L, validateAndParseTime("COL", "1674478926", 9, 0).longValueExact());
     assertEquals(
-        46926123456000L, validateAndParseTime("COL", "1674478926123456", 9).longValueExact());
+        46926123000000L, validateAndParseTime("COL", "1674478926123", 9, 0).longValueExact());
     assertEquals(
-        46926123456789L, validateAndParseTime("COL", "1674478926123456789", 9).longValueExact());
+        46926123456000L, validateAndParseTime("COL", "1674478926123456", 9, 0).longValueExact());
+    assertEquals(
+        46926123456789L, validateAndParseTime("COL", "1674478926123456789", 9, 0).longValueExact());
 
     // Test Java objects
     assertEquals(
         46926123456789L,
-        validateAndParseTime("COL", LocalTime.of(13, 2, 6, 123456789), 9).longValueExact());
+        validateAndParseTime("COL", LocalTime.of(13, 2, 6, 123456789), 9, 0).longValueExact());
     assertEquals(
         46926123456789L,
-        validateAndParseTime("COL", OffsetTime.of(13, 2, 6, 123456789, ZoneOffset.of("+09:00")), 9)
+        validateAndParseTime(
+                "COL", OffsetTime.of(13, 2, 6, 123456789, ZoneOffset.of("+09:00")), 9, 0)
             .longValueExact());
 
     // Dates and timestamps are forbidden
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2023-01-19", 9));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseTime("COL", "2023-01-19", 9, 0));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "2023-01-19T14:23:55.878137", 9));
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseTime("COL", "2023-01-19T14:23:55.878137", 9, 0));
 
     // Test forbidden values
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", LocalDate.now(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", LocalDateTime.now(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", OffsetDateTime.now(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", ZonedDateTime.now(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", Instant.now(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", new Date(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", 1.5f, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", 1.5, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "1.5", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "1.0", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", new Object(), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", false, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "", 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", "foo", 3));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("COL", java.sql.Time.valueOf("20:57:00"), 3));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("COL", java.sql.Date.valueOf("2010-11-03"), 3));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTime("COL", java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", BigInteger.ZERO, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", BigDecimal.ZERO, 3));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTime("COL", 'c', 3));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseTime("COL", LocalDate.now(), 3, 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseTime("COL", LocalDateTime.now(), 3, 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseTime("COL", OffsetDateTime.now(), 3, 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseTime("COL", ZonedDateTime.now(), 3, 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseTime("COL", Instant.now(), 3, 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseTime("COL", new Date(), 3, 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseTime("COL", 1.5f, 3, 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseTime("COL", 1.5, 3, 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseTime("COL", "1.5", 3, 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseTime("COL", "1.0", 3, 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseTime("COL", new Object(), 3, 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseTime("COL", false, 3, 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseTime("COL", "", 3, 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseTime("COL", "foo", 3, 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseTime("COL", java.sql.Time.valueOf("20:57:00"), 3, 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseTime("COL", java.sql.Date.valueOf("2010-11-03"), 3, 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseTime("COL", java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3, 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseTime("COL", BigInteger.ZERO, 3, 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseTime("COL", BigDecimal.ZERO, 3, 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseTime("COL", 'c', 3, 0));
   }
 
   @Test
   public void testValidateAndParseTimestamp() {
     TimestampWrapper wrapper =
         DataValidationUtil.validateAndParseTimestamp(
-            "COL", "2021-01-01T01:00:00.123+01:00", 4, UTC, false);
+            "COL", "2021-01-01T01:00:00.123+01:00", 4, UTC, false, 0);
     assertEquals(1609459200, wrapper.getEpoch());
     assertEquals(123000000, wrapper.getFraction());
     assertEquals(3600, wrapper.getTimezoneOffsetSeconds());
     assertEquals(1500, wrapper.getTimeZoneIndex());
 
-    wrapper = validateAndParseTimestamp("COL", "  2021-01-01T01:00:00.123 \t\n", 9, UTC, true);
+    wrapper = validateAndParseTimestamp("COL", "  2021-01-01T01:00:00.123 \t\n", 9, UTC, true, 0);
     Assert.assertEquals(1609462800, wrapper.getEpoch());
     Assert.assertEquals(123000000, wrapper.getFraction());
     Assert.assertEquals(new BigInteger("1609462800123000000"), wrapper.toBinary(false));
 
     // Time input is not supported
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "20:57:01", 3, UTC, false));
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseTimestamp("COL", "20:57:01", 3, UTC, false, 0));
 
     // Test forbidden values
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestamp("COL", LocalTime.now(), 3, UTC, false));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseTimestamp("COL", LocalTime.now(), 3, UTC, false, 0));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestamp("COL", OffsetTime.now(), 3, UTC, false));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseTimestamp("COL", OffsetTime.now(), 3, UTC, false, 0));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", new Date(), 3, UTC, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 1.5f, 3, UTC, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 1.5, 3, UTC, false));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseTimestamp("COL", new Date(), 3, UTC, false, 0));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "1.5", 3, UTC, false));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseTimestamp("COL", 1.5f, 3, UTC, false, 0));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "1.0", 3, UTC, false));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseTimestamp("COL", 1.5, 3, UTC, false, 0));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, false));
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseTimestamp("COL", "1.5", 3, UTC, false, 0));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", false, 3, UTC, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "", 3, UTC, false));
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseTimestamp("COL", "1.0", 3, UTC, false, 0));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", "foo", 3, UTC, false));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, false, 0));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestamp("COL", java.sql.Time.valueOf("20:57:00"), 3, UTC, false));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseTimestamp("COL", false, 3, UTC, false, 0));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestamp("COL", java.sql.Date.valueOf("2010-11-03"), 3, UTC, false));
+        ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseTimestamp("COL", "", 3, UTC, false, 0));
     expectError(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseTimestamp("COL", "foo", 3, UTC, false, 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW,
+        () ->
+            validateAndParseTimestamp("COL", java.sql.Time.valueOf("20:57:00"), 3, UTC, false, 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW,
         () ->
             validateAndParseTimestamp(
-                "COL", java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3, UTC, false));
+                "COL", java.sql.Date.valueOf("2010-11-03"), 3, UTC, false, 0));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestamp("COL", BigInteger.ZERO, 3, UTC, false));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () ->
+            validateAndParseTimestamp(
+                "COL", java.sql.Timestamp.valueOf("2010-11-03 20:57:00"), 3, UTC, false, 0));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseTimestamp("COL", BigDecimal.ZERO, 3, UTC, false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseTimestamp("COL", 'c', 3, UTC, false));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseTimestamp("COL", BigInteger.ZERO, 3, UTC, false, 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseTimestamp("COL", BigDecimal.ZERO, 3, UTC, false, 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseTimestamp("COL", 'c', 3, UTC, false, 0));
   }
 
   @Test
   public void testValidateAndParseBigDecimal() {
-    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal("COL", "1"));
-    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal("COL", "  1 \t\n "));
+    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal("COL", "1", 0));
+    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal("COL", "  1 \t\n ", 0));
     assertEquals(
         new BigDecimal("1000").toBigInteger(),
-        validateAndParseBigDecimal("COL", "1e3").toBigInteger());
+        validateAndParseBigDecimal("COL", "1e3", 0).toBigInteger());
     assertEquals(
         new BigDecimal("1000").toBigInteger(),
-        validateAndParseBigDecimal("COL", "  1e3 \t\n").toBigInteger());
+        validateAndParseBigDecimal("COL", "  1e3 \t\n", 0).toBigInteger());
     assertEquals(
         new BigDecimal("1000").toBigInteger(),
-        validateAndParseBigDecimal("COL", "1e3").toBigInteger());
+        validateAndParseBigDecimal("COL", "1e3", 0).toBigInteger());
     assertEquals(
         new BigDecimal("-1000").toBigInteger(),
-        validateAndParseBigDecimal("COL", "-1e3").toBigInteger());
+        validateAndParseBigDecimal("COL", "-1e3", 0).toBigInteger());
     assertEquals(
         new BigDecimal("1").toBigInteger(),
-        validateAndParseBigDecimal("COL", "1e0").toBigInteger());
+        validateAndParseBigDecimal("COL", "1e0", 0).toBigInteger());
     assertEquals(
         new BigDecimal("-1").toBigInteger(),
-        validateAndParseBigDecimal("COL", "-1e0").toBigInteger());
+        validateAndParseBigDecimal("COL", "-1e0", 0).toBigInteger());
     assertEquals(
         new BigDecimal("123").toBigInteger(),
-        validateAndParseBigDecimal("COL", "1.23e2").toBigInteger());
+        validateAndParseBigDecimal("COL", "1.23e2", 0).toBigInteger());
     assertEquals(
         new BigDecimal("123.4").toBigInteger(),
-        validateAndParseBigDecimal("COL", "1.234e2").toBigInteger());
+        validateAndParseBigDecimal("COL", "1.234e2", 0).toBigInteger());
     assertEquals(
         new BigDecimal("0.1234").toBigInteger(),
-        validateAndParseBigDecimal("COL", "1.234e-1").toBigInteger());
+        validateAndParseBigDecimal("COL", "1.234e-1", 0).toBigInteger());
     assertEquals(
         new BigDecimal("0.1234").toBigInteger(),
-        validateAndParseBigDecimal("COL", "1234e-5").toBigInteger());
+        validateAndParseBigDecimal("COL", "1234e-5", 0).toBigInteger());
     assertEquals(
         new BigDecimal("0.1234").toBigInteger(),
-        validateAndParseBigDecimal("COL", "1234E-5").toBigInteger());
-    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal("COL", 1));
-    assertEquals(new BigDecimal("1.0"), validateAndParseBigDecimal("COL", 1D));
-    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal("COL", 1L));
-    assertEquals(new BigDecimal("1.0"), validateAndParseBigDecimal("COL", 1F));
+        validateAndParseBigDecimal("COL", "1234E-5", 0).toBigInteger());
+    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal("COL", 1, 0));
+    assertEquals(new BigDecimal("1.0"), validateAndParseBigDecimal("COL", 1D, 0));
+    assertEquals(new BigDecimal("1"), validateAndParseBigDecimal("COL", 1L, 0));
+    assertEquals(new BigDecimal("1.0"), validateAndParseBigDecimal("COL", 1F, 0));
     assertEquals(
         BigDecimal.valueOf(10).pow(37),
-        validateAndParseBigDecimal("COL", BigDecimal.valueOf(10).pow(37)));
+        validateAndParseBigDecimal("COL", BigDecimal.valueOf(10).pow(37), 0));
     assertEquals(
         BigDecimal.valueOf(-1).multiply(BigDecimal.valueOf(10).pow(37)),
         validateAndParseBigDecimal(
-            "COL", BigInteger.valueOf(-1).multiply(BigInteger.valueOf(10).pow(37))));
+            "COL", BigInteger.valueOf(-1).multiply(BigInteger.valueOf(10).pow(37)), 0));
 
     // Test forbidden values
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBigDecimal("COL", "honk"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBigDecimal("COL", "0x22"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBigDecimal("COL", true));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBigDecimal("COL", false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBigDecimal("COL", new Object()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBigDecimal("COL", 'a'));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBigDecimal("COL", new byte[4]));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseBigDecimal("COL", "honk", 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseBigDecimal("COL", "0x22", 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseBigDecimal("COL", true, 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseBigDecimal("COL", false, 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseBigDecimal("COL", new Object(), 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseBigDecimal("COL", 'a', 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseBigDecimal("COL", new byte[4], 0));
   }
 
   @Test
   public void testValidateAndParseString() {
-    assertEquals("honk", validateAndParseString("COL", "honk", Optional.empty()));
+    assertEquals("honk", validateAndParseString("COL", "honk", Optional.empty(), 0));
 
     // Check max byte length
     String maxString = buildString("a", BYTES_16_MB);
-    assertEquals(maxString, validateAndParseString("COL", maxString, Optional.empty()));
+    assertEquals(maxString, validateAndParseString("COL", maxString, Optional.empty(), 0));
 
     // max byte length - 1 should also succeed
     String maxStringMinusOne = buildString("a", BYTES_16_MB - 1);
     assertEquals(
-        maxStringMinusOne, validateAndParseString("COL", maxStringMinusOne, Optional.empty()));
+        maxStringMinusOne, validateAndParseString("COL", maxStringMinusOne, Optional.empty(), 0));
 
     // max byte length + 1 should fail
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseString("COL", maxString + "a", Optional.empty()));
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseString("COL", maxString + "a", Optional.empty(), 0));
 
     // Test that max character length validation counts characters and not bytes
-    assertEquals("a", validateAndParseString("COL", "a", Optional.of(1)));
-    assertEquals("č", validateAndParseString("COL", "č", Optional.of(1)));
-    assertEquals("❄", validateAndParseString("COL", "❄", Optional.of(1)));
-    assertEquals("🍞", validateAndParseString("COL", "🍞", Optional.of(1)));
+    assertEquals("a", validateAndParseString("COL", "a", Optional.of(1), 0));
+    assertEquals("č", validateAndParseString("COL", "č", Optional.of(1), 0));
+    assertEquals("❄", validateAndParseString("COL", "❄", Optional.of(1), 0));
+    assertEquals("🍞", validateAndParseString("COL", "🍞", Optional.of(1), 0));
 
     // Test max character length rejection
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", "a🍞", Optional.of(1)));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", "12345", Optional.of(4)));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", false, Optional.of(4)));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", 12345, Optional.of(4)));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", 1.2345, Optional.of(4)));
+        ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseString("COL", "a🍞", Optional.of(1), 0));
+    expectError(
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseString("COL", "12345", Optional.of(4), 0));
+    expectError(
+        ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseString("COL", false, Optional.of(4), 0));
+    expectError(
+        ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseString("COL", 12345, Optional.of(4), 0));
+    expectError(
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseString("COL", 1.2345, Optional.of(4), 0));
 
     // Test that invalid UTF-8 strings cannot be ingested
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseString("COL", "foo\uD800bar", Optional.empty()));
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseString("COL", "foo\uD800bar", Optional.empty(), 0));
 
     // Test unsupported values
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", new Object(), Optional.empty()));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseString("COL", new Object(), Optional.empty(), 0));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", new byte[] {}, Optional.of(4)));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseString("COL", new byte[] {}, Optional.of(4), 0));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseString("COL", new char[] {}, Optional.of(4)));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseString("COL", new char[] {}, Optional.of(4), 0));
   }
 
   @Test
   public void testValidateAndParseVariant() throws Exception {
-    assertEquals("1", validateAndParseVariant("COL", 1));
-    assertEquals("1", validateAndParseVariant("COL", "1"));
-    assertEquals("1", validateAndParseVariant("COL", "                          1   "));
+    assertEquals("1", validateAndParseVariant("COL", 1, 0));
+    assertEquals("1", validateAndParseVariant("COL", "1", 0));
+    assertEquals("1", validateAndParseVariant("COL", "                          1   ", 0));
     String stringVariant = "{\"key\":1}";
-    assertEquals(stringVariant, validateAndParseVariant("COL", stringVariant));
-    assertEquals(stringVariant, validateAndParseVariant("COL", "  " + stringVariant + " \t\n"));
+    assertEquals(stringVariant, validateAndParseVariant("COL", stringVariant, 0));
+    assertEquals(stringVariant, validateAndParseVariant("COL", "  " + stringVariant + " \t\n", 0));
 
     // Test custom serializers
     assertEquals(
         "[-128,0,127]",
-        validateAndParseVariant("COL", new byte[] {Byte.MIN_VALUE, 0, Byte.MAX_VALUE}));
+        validateAndParseVariant("COL", new byte[] {Byte.MIN_VALUE, 0, Byte.MAX_VALUE}, 0));
     assertEquals(
         "\"2022-09-28T03:04:12.123456789-07:00\"",
         validateAndParseVariant(
             "COL",
-            ZonedDateTime.of(2022, 9, 28, 3, 4, 12, 123456789, ZoneId.of("America/Los_Angeles"))));
+            ZonedDateTime.of(2022, 9, 28, 3, 4, 12, 123456789, ZoneId.of("America/Los_Angeles")),
+            0));
 
     // Test valid JSON tokens
-    assertEquals("null", validateAndParseVariant("COL", null));
-    assertEquals("null", validateAndParseVariant("COL", "null"));
-    assertEquals("true", validateAndParseVariant("COL", true));
-    assertEquals("true", validateAndParseVariant("COL", "true"));
-    assertEquals("false", validateAndParseVariant("COL", false));
-    assertEquals("false", validateAndParseVariant("COL", "false"));
-    assertEquals("{}", validateAndParseVariant("COL", "{}"));
-    assertEquals("[]", validateAndParseVariant("COL", "[]"));
-    assertEquals("[\"foo\",1,null]", validateAndParseVariant("COL", "[\"foo\",1,null]"));
-    assertEquals("\"\"", validateAndParseVariant("COL", "\"\""));
+    assertEquals("null", validateAndParseVariant("COL", null, 0));
+    assertEquals("null", validateAndParseVariant("COL", "null", 0));
+    assertEquals("true", validateAndParseVariant("COL", true, 0));
+    assertEquals("true", validateAndParseVariant("COL", "true", 0));
+    assertEquals("false", validateAndParseVariant("COL", false, 0));
+    assertEquals("false", validateAndParseVariant("COL", "false", 0));
+    assertEquals("{}", validateAndParseVariant("COL", "{}", 0));
+    assertEquals("[]", validateAndParseVariant("COL", "[]", 0));
+    assertEquals("[\"foo\",1,null]", validateAndParseVariant("COL", "[\"foo\",1,null]", 0));
+    assertEquals("\"\"", validateAndParseVariant("COL", "\"\"", 0));
 
     // Test missing values are null instead of empty string
-    assertNull(validateAndParseVariant("COL", ""));
-    assertNull(validateAndParseVariant("COL", "  "));
+    assertNull(validateAndParseVariant("COL", "", 0));
+    assertNull(validateAndParseVariant("COL", "  ", 0));
 
     // Test that invalid UTF-8 strings cannot be ingested
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", "\"foo\uD800bar\""));
+    expectError(
+        ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseVariant("COL", "\"foo\uD800bar\"", 0));
 
     // Test forbidden values
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", "{null}"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", "}{"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", readTree("{}")));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", new Object()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", "foo"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", new Date()));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseVariant("COL", "{null}", 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseVariant("COL", "}{", 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseVariant("COL", readTree("{}"), 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseVariant("COL", new Object(), 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseVariant("COL", "foo", 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseVariant("COL", new Date(), 0));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseVariant("COL", Collections.singletonList(new Object())));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseVariant("COL", Collections.singletonList(new Object()), 0));
     expectError(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         () ->
             validateAndParseVariant(
-                "COL", Collections.singletonList(Collections.singletonMap("foo", new Object()))));
+                "COL",
+                Collections.singletonList(Collections.singletonMap("foo", new Object())),
+                0));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseVariant("COL", Collections.singletonMap(new Object(), "foo")));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseVariant("COL", Collections.singletonMap(new Object(), "foo"), 0));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseVariant("COL", Collections.singletonMap("foo", new Object())));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseVariant("COL", Collections.singletonMap("foo", new Object()), 0));
   }
 
   @Test
   public void testValidateAndParseArray() throws Exception {
-    assertEquals("[1]", validateAndParseArray("COL", 1));
-    assertEquals("[1]", validateAndParseArray("COL", "1"));
-    assertEquals("[1]", validateAndParseArray("COL", "                          1   "));
-    assertEquals("[1,2,3]", validateAndParseArray("COL", "[1, 2, 3]"));
-    assertEquals("[1,2,3]", validateAndParseArray("COL", "  [1, 2, 3] \t\n"));
+    assertEquals("[1]", validateAndParseArray("COL", 1, 0));
+    assertEquals("[1]", validateAndParseArray("COL", "1", 0));
+    assertEquals("[1]", validateAndParseArray("COL", "                          1   ", 0));
+    assertEquals("[1,2,3]", validateAndParseArray("COL", "[1, 2, 3]", 0));
+    assertEquals("[1,2,3]", validateAndParseArray("COL", "  [1, 2, 3] \t\n", 0));
     int[] intArray = new int[] {1, 2, 3};
-    assertEquals("[1,2,3]", validateAndParseArray("COL", intArray));
+    assertEquals("[1,2,3]", validateAndParseArray("COL", intArray, 0));
 
     String[] stringArray = new String[] {"a", "b", "c"};
-    assertEquals("[\"a\",\"b\",\"c\"]", validateAndParseArray("COL", stringArray));
+    assertEquals("[\"a\",\"b\",\"c\"]", validateAndParseArray("COL", stringArray, 0));
 
     Object[] objectArray = new Object[] {1, 2, 3};
-    assertEquals("[1,2,3]", validateAndParseArray("COL", objectArray));
+    assertEquals("[1,2,3]", validateAndParseArray("COL", objectArray, 0));
 
     Object[] ObjectArrayWithNull = new Object[] {1, null, 3};
-    assertEquals("[1,null,3]", validateAndParseArray("COL", ObjectArrayWithNull));
+    assertEquals("[1,null,3]", validateAndParseArray("COL", ObjectArrayWithNull, 0));
 
     Object[][] nestedArray = new Object[][] {{1, 2, 3}, null, {4, 5, 6}};
-    assertEquals("[[1,2,3],null,[4,5,6]]", validateAndParseArray("COL", nestedArray));
+    assertEquals("[[1,2,3],null,[4,5,6]]", validateAndParseArray("COL", nestedArray, 0));
 
     List<Integer> intList = Arrays.asList(1, 2, 3);
-    assertEquals("[1,2,3]", validateAndParseArray("COL", intList));
+    assertEquals("[1,2,3]", validateAndParseArray("COL", intList, 0));
 
     List<Object> objectList = Arrays.asList(1, 2, 3);
-    assertEquals("[1,2,3]", validateAndParseArray("COL", objectList));
+    assertEquals("[1,2,3]", validateAndParseArray("COL", objectList, 0));
 
     List<Object> nestedList = Arrays.asList(Arrays.asList(1, 2, 3), 2, 3);
-    assertEquals("[[1,2,3],2,3]", validateAndParseArray("COL", nestedList));
+    assertEquals("[[1,2,3],2,3]", validateAndParseArray("COL", nestedList, 0));
 
     // Test null values
-    assertEquals("[null]", validateAndParseArray("COL", ""));
-    assertEquals("[null]", validateAndParseArray("COL", " "));
-    assertEquals("[null]", validateAndParseArray("COL", "null"));
-    assertEquals("[null]", validateAndParseArray("COL", null));
+    assertEquals("[null]", validateAndParseArray("COL", "", 0));
+    assertEquals("[null]", validateAndParseArray("COL", " ", 0));
+    assertEquals("[null]", validateAndParseArray("COL", "null", 0));
+    assertEquals("[null]", validateAndParseArray("COL", null, 0));
 
     // Test that invalid UTF-8 strings cannot be ingested
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseArray("COL", "\"foo\uD800bar\""));
+    expectError(
+        ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseArray("COL", "\"foo\uD800bar\"", 0));
 
     // Test forbidden values
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseArray("COL", readTree("[]")));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseArray("COL", new Object()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseArray("COL", "foo")); // invalid JSO)N
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseArray("COL", new Date()));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseArray("COL", Collections.singletonList(new Object())));
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseArray("COL", readTree("[]"), 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseArray("COL", new Object(), 0));
+    expectError(
+        ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseArray("COL", "foo", 0)); // invalid JSO)N
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseArray("COL", new Date(), 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseArray("COL", Collections.singletonList(new Object()), 0));
     expectError(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         () ->
             validateAndParseArray(
-                "COL", Collections.singletonList(Collections.singletonMap("foo", new Object()))));
+                "COL",
+                Collections.singletonList(Collections.singletonMap("foo", new Object())),
+                0));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseArray("COL", Collections.singletonMap(new Object(), "foo")));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseArray("COL", Collections.singletonMap(new Object(), "foo"), 0));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseArray("COL", Collections.singletonMap("foo", new Object())));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseArray("COL", Collections.singletonMap("foo", new Object()), 0));
   }
 
   @Test
   public void testValidateAndParseObject() throws Exception {
     String stringObject = "{\"key\":1}";
-    assertEquals(stringObject, validateAndParseObject("COL", stringObject));
-    assertEquals(stringObject, validateAndParseObject("COL", "  " + stringObject + " \t\n"));
+    assertEquals(stringObject, validateAndParseObject("COL", stringObject, 0));
+    assertEquals(stringObject, validateAndParseObject("COL", "  " + stringObject + " \t\n", 0));
 
     String badObject = "foo";
     try {
-      validateAndParseObject("COL", badObject);
+      validateAndParseObject("COL", badObject, 0);
       Assert.fail("Expected INVALID_ROW error");
     } catch (SFException err) {
-      assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), err.getVendorCode());
+      assertEquals(ErrorCode.INVALID_VALUE_ROW.getMessageCode(), err.getVendorCode());
     }
 
     char[] data = new char[20000000];
@@ -526,40 +590,44 @@ public void testValidateAndParseObject() throws Exception {
     mapVal.put("key", stringVal);
     String tooLargeObject = objectMapper.writeValueAsString(mapVal);
     try {
-      validateAndParseObject("COL", tooLargeObject);
+      validateAndParseObject("COL", tooLargeObject, 0);
       Assert.fail("Expected INVALID_ROW error");
     } catch (SFException err) {
-      assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), err.getVendorCode());
+      assertEquals(ErrorCode.INVALID_VALUE_ROW.getMessageCode(), err.getVendorCode());
     }
 
     // Test that invalid UTF-8 strings cannot be ingested
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", "{\"foo\": \"foo\uD800bar\"}"));
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseObject("COL", "{\"foo\": \"foo\uD800bar\"}", 0));
 
     // Test forbidden values
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", readTree("{}")));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", "[]"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", "1"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", 1));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", 1.5));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", new Object()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", "foo"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", new Date()));
-    expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseObject("COL", Collections.singletonList(new Object())));
-    expectError(
-        ErrorCode.INVALID_ROW,
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseObject("COL", readTree("{}"), 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseObject("COL", "[]", 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseObject("COL", "1", 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseObject("COL", 1, 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseObject("COL", 1.5, 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseObject("COL", false, 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseObject("COL", new Object(), 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseObject("COL", "foo", 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseObject("COL", new Date(), 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseObject("COL", Collections.singletonList(new Object()), 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW,
         () ->
             validateAndParseObject(
-                "COL", Collections.singletonList(Collections.singletonMap("foo", new Object()))));
+                "COL",
+                Collections.singletonList(Collections.singletonMap("foo", new Object())),
+                0));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseObject("COL", Collections.singletonMap(new Object(), "foo")));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseObject("COL", Collections.singletonMap(new Object(), "foo"), 0));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseObject("COL", Collections.singletonMap("foo", new Object())));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseObject("COL", Collections.singletonMap("foo", new Object()), 0));
   }
 
   @Test
@@ -571,9 +639,9 @@ public void testTooLargeVariant() {
     Map<String, Object> m = new HashMap<>();
     m.put("a", "11");
     m.put("b", new String(stringContent));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseVariant("COL", m));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseArray("COL", m));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseObject("COL", m));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseVariant("COL", m, 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseArray("COL", m, 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseObject("COL", m, 0));
   }
 
   @Test
@@ -585,23 +653,23 @@ public void testTooLargeMultiByteSemiStructuredValues() {
     Map<String, Object> m = new HashMap<>();
     m.put("a", new String(stringContent));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: {\"a\":\"ČČČČČČČČČČČČČČ...."
-            + " Value cannot be ingested into Snowflake column COL of type VARIANT: Variant too"
-            + " long: length=18874376 maxLength=16777152",
-        () -> validateAndParseVariant("COL", m));
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type VARIANT, Row Index: 0, reason:"
+            + " Variant too long: length=18874376 maxLength=16777152",
+        () -> validateAndParseVariant("COL", m, 0));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: [{\"a\":\"ČČČČČČČČČČČČČ...."
-            + " Value cannot be ingested into Snowflake column COL of type ARRAY: Array too large."
-            + " length=18874378 maxLength=16777152",
-        () -> validateAndParseArray("COL", m));
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type ARRAY, Row Index: 0, reason:"
+            + " Array too large. length=18874378 maxLength=16777152",
+        () -> validateAndParseArray("COL", m, 0));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: {\"a\":\"ČČČČČČČČČČČČČČ...."
-            + " Value cannot be ingested into Snowflake column COL of type OBJECT: Object too"
-            + " large. length=18874376 maxLength=16777152",
-        () -> validateAndParseObject("COL", m));
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type OBJECT, Row Index: 0, reason:"
+            + " Object too large. length=18874376 maxLength=16777152",
+        () -> validateAndParseObject("COL", m, 0));
   }
 
   @Test
@@ -767,85 +835,103 @@ public void testValidateAndParseBinary() throws DecoderException {
 
     assertArrayEquals(
         "honk".getBytes(StandardCharsets.UTF_8),
-        validateAndParseBinary("COL", "honk".getBytes(StandardCharsets.UTF_8), Optional.empty()));
+        validateAndParseBinary(
+            "COL", "honk".getBytes(StandardCharsets.UTF_8), Optional.empty(), 0));
 
     assertArrayEquals(
         new byte[] {-1, 0, 1},
-        validateAndParseBinary("COL", new byte[] {-1, 0, 1}, Optional.empty()));
+        validateAndParseBinary("COL", new byte[] {-1, 0, 1}, Optional.empty(), 0));
     assertArrayEquals(
         Hex.decodeHex("1234567890abcdef"), // pragma: allowlist secret NOT A SECRET
         validateAndParseBinary(
-            "COL", "1234567890abcdef", Optional.empty())); // pragma: allowlist secret NOT A SECRET
+            "COL",
+            "1234567890abcdef",
+            Optional.empty(),
+            0)); // pragma: allowlist secret NOT A SECRET
     assertArrayEquals(
         Hex.decodeHex("1234567890abcdef"), // pragma: allowlist secret NOT A SECRET
         validateAndParseBinary(
             "COL",
             "  1234567890abcdef \t\n",
-            Optional.empty())); // pragma: allowlist secret NOT A SECRET
+            Optional.empty(),
+            0)); // pragma: allowlist secret NOT A SECRET
 
     assertArrayEquals(
-        maxAllowedArray, validateAndParseBinary("COL", maxAllowedArray, Optional.empty()));
+        maxAllowedArray, validateAndParseBinary("COL", maxAllowedArray, Optional.empty(), 0));
     assertArrayEquals(
         maxAllowedArrayMinusOne,
-        validateAndParseBinary("COL", maxAllowedArrayMinusOne, Optional.empty()));
+        validateAndParseBinary("COL", maxAllowedArrayMinusOne, Optional.empty(), 0));
 
     // Too large arrays should be rejected
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", new byte[1], Optional.of(0)));
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseBinary("COL", new byte[1], Optional.of(0), 0));
+    expectError(
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseBinary("COL", new byte[BYTES_8_MB + 1], Optional.empty(), 0));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseBinary("COL", new byte[BYTES_8_MB + 1], Optional.empty()));
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseBinary("COL", new byte[8], Optional.of(7), 0));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", new byte[8], Optional.of(7)));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", "aabb", Optional.of(1)));
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseBinary("COL", "aabb", Optional.of(1), 0));
 
     // unsupported data types should fail
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", "000", Optional.empty()));
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseBinary("COL", "000", Optional.empty(), 0));
+    expectError(
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseBinary("COL", "abcg", Optional.empty(), 0));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", "abcg", Optional.empty()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", "c", Optional.empty()));
+        ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseBinary("COL", "c", Optional.empty(), 0));
     expectError(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         () ->
             validateAndParseBinary(
-                "COL", Arrays.asList((byte) 1, (byte) 2, (byte) 3), Optional.empty()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", 1, Optional.empty()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", 12, Optional.empty()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", 1.5, Optional.empty()));
+                "COL", Arrays.asList((byte) 1, (byte) 2, (byte) 3), Optional.empty(), 0));
     expectError(
-        ErrorCode.INVALID_ROW,
-        () -> validateAndParseBinary("COL", BigInteger.ONE, Optional.empty()));
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseBinary("COL", 1, Optional.empty(), 0));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", false, Optional.empty()));
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseBinary("COL", 12, Optional.empty(), 0));
     expectError(
-        ErrorCode.INVALID_ROW, () -> validateAndParseBinary("COL", new Object(), Optional.empty()));
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseBinary("COL", 1.5, Optional.empty(), 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseBinary("COL", BigInteger.ONE, Optional.empty(), 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseBinary("COL", false, Optional.empty(), 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW,
+        () -> validateAndParseBinary("COL", new Object(), Optional.empty(), 0));
   }
 
   @Test
   public void testValidateAndParseReal() throws Exception {
     // From number types
-    assertEquals(1.23d, validateAndParseReal("COL", 1.23f), 0);
-    assertEquals(1.23d, validateAndParseReal("COL", 1.23), 0);
-    assertEquals(1.23d, validateAndParseReal("COL", 1.23d), 0);
-    assertEquals(1.23d, validateAndParseReal("COL", new BigDecimal("1.23")), 0);
-    assertEquals(Double.NaN, validateAndParseReal("COL", "Nan"), 0);
-    assertEquals(Double.POSITIVE_INFINITY, validateAndParseReal("COL", "inF"), 0);
-    assertEquals(Double.NEGATIVE_INFINITY, validateAndParseReal("COL", "-inF"), 0);
-    assertEquals(Double.NEGATIVE_INFINITY, validateAndParseReal("COL", " -inF \t\n"), 0);
+    assertEquals(1.23d, validateAndParseReal("COL", 1.23f, 0), 0);
+    assertEquals(1.23d, validateAndParseReal("COL", 1.23, 0), 0);
+    assertEquals(1.23d, validateAndParseReal("COL", 1.23d, 0), 0);
+    assertEquals(1.23d, validateAndParseReal("COL", new BigDecimal("1.23"), 0), 0);
+    assertEquals(Double.NaN, validateAndParseReal("COL", "Nan", 0), 0);
+    assertEquals(Double.POSITIVE_INFINITY, validateAndParseReal("COL", "inF", 0), 0);
+    assertEquals(Double.NEGATIVE_INFINITY, validateAndParseReal("COL", "-inF", 0), 0);
+    assertEquals(Double.NEGATIVE_INFINITY, validateAndParseReal("COL", " -inF \t\n", 0), 0);
 
     // From string
-    assertEquals(1.23d, validateAndParseReal("COL", "   1.23 \t\n"), 0);
-    assertEquals(1.23d, validateAndParseReal("COL", "1.23"), 0);
-    assertEquals(123d, validateAndParseReal("COL", "1.23E2"), 0);
-    assertEquals(123d, validateAndParseReal("COL", "1.23e2"), 0);
+    assertEquals(1.23d, validateAndParseReal("COL", "   1.23 \t\n", 0), 0);
+    assertEquals(1.23d, validateAndParseReal("COL", "1.23", 0), 0);
+    assertEquals(123d, validateAndParseReal("COL", "1.23E2", 0), 0);
+    assertEquals(123d, validateAndParseReal("COL", "1.23e2", 0), 0);
 
     // Test forbidden values
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseReal("COL", "foo"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseReal("COL", 'c'));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseReal("COL", new Object()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseReal("COL", false));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseReal("COL", true));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseReal("COL", "foo", 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseReal("COL", 'c', 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseReal("COL", new Object(), 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseReal("COL", false, 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseReal("COL", true, 0));
   }
 
   @Test
@@ -868,21 +954,24 @@ public void testValidateAndParseBoolean() {
             -1.1,
             -10,
             10)) {
-      assertEquals(1, validateAndParseBoolean("COL", input));
+      assertEquals(1, validateAndParseBoolean("COL", input, 0));
     }
-
+    int rowIndex = 0;
     for (Object input :
         Arrays.asList(false, "false", "False", "FalsE", "f", "no", "NO", "n", "off", "0", 0)) {
-      assertEquals(0, validateAndParseBoolean("COL", input));
+      assertEquals(0, validateAndParseBoolean("COL", input, rowIndex));
+      rowIndex += 1;
     }
 
     // Test forbidden values
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBoolean("COL", new Object()));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBoolean("COL", 't'));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBoolean("COL", 'f'));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBoolean("COL", new int[] {}));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBoolean("COL", "foobar"));
-    expectError(ErrorCode.INVALID_ROW, () -> validateAndParseBoolean("COL", ""));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseBoolean("COL", new Object(), 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseBoolean("COL", 't', 0));
+    expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseBoolean("COL", 'f', 0));
+    expectError(
+        ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseBoolean("COL", new int[] {}, 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseBoolean("COL", "foobar", 0));
+    expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseBoolean("COL", "", 0));
   }
 
   /**
@@ -893,194 +982,209 @@ public void testValidateAndParseBoolean() {
   public void testExceptionMessages() {
     // BOOLEAN
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type BOOLEAN. Allowed Java types:"
-            + " boolean, Number, String",
-        () -> validateAndParseBoolean("COL", new Object()));
+            + " cannot be ingested into Snowflake column COL of type BOOLEAN, Row Index:0. Allowed"
+            + " Java types: boolean, Number, String",
+        () -> validateAndParseBoolean("COL", new Object(), 0));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
-            + " into Snowflake column COL of type BOOLEAN: Not a valid boolean, see"
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type BOOLEAN, Row Index: 0, reason:"
+            + " Not a valid boolean, see"
             + " https://docs.snowflake.com/en/sql-reference/data-types-logical.html#conversion-to-boolean"
             + " for the list of supported formats",
-        () -> validateAndParseBoolean("COL", "abc"));
+        () -> validateAndParseBoolean("COL", "abc", 0));
 
     // TIME
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type TIME. Allowed Java types:"
-            + " String, LocalTime, OffsetTime",
-        () -> validateAndParseTime("COL", new Object(), 10));
+            + " cannot be ingested into Snowflake column COL of type TIME, Row Index:0. Allowed"
+            + " Java types: String, LocalTime, OffsetTime",
+        () -> validateAndParseTime("COL", new Object(), 10, 0));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
-            + " into Snowflake column COL of type TIME: Not a valid time, see"
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type TIME, Row Index: 0, reason:"
+            + " Not a valid time, see"
             + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
             + " the list of supported formats",
-        () -> validateAndParseTime("COL", "abc", 10));
+        () -> validateAndParseTime("COL", "abc", 10, 0));
 
     // DATE
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type DATE. Allowed Java types:"
-            + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
-        () -> validateAndParseDate("COL", new Object()));
+            + " cannot be ingested into Snowflake column COL of type DATE, Row Index:0. Allowed"
+            + " Java types: String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
+        () -> validateAndParseDate("COL", new Object(), 0));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
-            + " into Snowflake column COL of type DATE: Not a valid value, see"
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type DATE, Row Index: 0, reason:"
+            + " Not a valid value, see"
             + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
             + " the list of supported formats",
-        () -> validateAndParseDate("COL", "abc"));
+        () -> validateAndParseDate("COL", "abc", 0));
 
     // TIMESTAMP_NTZ
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
-            + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
-        () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, true));
+            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, Row Index:0."
+            + " Allowed Java types: String, LocalDate, LocalDateTime, ZonedDateTime,"
+            + " OffsetDateTime",
+        () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, true, 0));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
-            + " into Snowflake column COL of type TIMESTAMP: Not a valid value, see"
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, Row Index: 0,"
+            + " reason: Not a valid value, see"
             + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
             + " the list of supported formats",
-        () -> validateAndParseTimestamp("COL", "abc", 3, UTC, true));
+        () -> validateAndParseTimestamp("COL", "abc", 3, UTC, true, 0));
 
     // TIMESTAMP_LTZ
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
-            + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
-        () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, false));
+            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, Row Index:0."
+            + " Allowed Java types: String, LocalDate, LocalDateTime, ZonedDateTime,"
+            + " OffsetDateTime",
+        () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, false, 0));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
-            + " into Snowflake column COL of type TIMESTAMP: Not a valid value, see"
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, Row Index: 0,"
+            + " reason: Not a valid value, see"
             + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
             + " the list of supported formats",
-        () -> validateAndParseTimestamp("COL", "abc", 3, UTC, false));
+        () -> validateAndParseTimestamp("COL", "abc", 3, UTC, false, 0));
 
     // TIMESTAMP_TZ
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type TIMESTAMP. Allowed Java types:"
-            + " String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
-        () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, false));
+            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, Row Index:0."
+            + " Allowed Java types: String, LocalDate, LocalDateTime, ZonedDateTime,"
+            + " OffsetDateTime",
+        () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, false, 0));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
-            + " into Snowflake column COL of type TIMESTAMP: Not a valid value, see"
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, Row Index: 0,"
+            + " reason: Not a valid value, see"
             + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
             + " the list of supported formats",
-        () -> validateAndParseTimestamp("COL", "abc", 3, UTC, false));
+        () -> validateAndParseTimestamp("COL", "abc", 3, UTC, false, 0));
 
     // NUMBER
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type NUMBER. Allowed Java types:"
-            + " int, long, byte, short, float, double, BigDecimal, BigInteger, String",
-        () -> validateAndParseBigDecimal("COL", new Object()));
+            + " cannot be ingested into Snowflake column COL of type NUMBER, Row Index:0. Allowed"
+            + " Java types: int, long, byte, short, float, double, BigDecimal, BigInteger, String",
+        () -> validateAndParseBigDecimal("COL", new Object(), 0));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
-            + " into Snowflake column COL of type NUMBER: Not a valid number",
-        () -> validateAndParseBigDecimal("COL", "abc"));
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type NUMBER, Row Index: 0, reason:"
+            + " Not a valid number",
+        () -> validateAndParseBigDecimal("COL", "abc", 0));
 
     // REAL
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type REAL. Allowed Java types:"
-            + " Number, String",
-        () -> validateAndParseReal("COL", new Object()));
+            + " cannot be ingested into Snowflake column COL of type REAL, Row Index:0. Allowed"
+            + " Java types: Number, String",
+        () -> validateAndParseReal("COL", new Object(), 0));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
-            + " into Snowflake column COL of type REAL: Not a valid decimal number",
-        () -> validateAndParseReal("COL", "abc"));
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type REAL, Row Index: 0, reason:"
+            + " Not a valid decimal number",
+        () -> validateAndParseReal("COL", "abc", 0));
 
     // STRING
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type STRING. Allowed Java types:"
-            + " String, Number, boolean, char",
-        () -> validateAndParseString("COL", new Object(), Optional.empty()));
+            + " cannot be ingested into Snowflake column COL of type STRING, Row Index:0. Allowed"
+            + " Java types: String, Number, boolean, char",
+        () -> validateAndParseString("COL", new Object(), Optional.empty(), 0));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: abc. Value cannot be ingested"
-            + " into Snowflake column COL of type STRING: String too long: length=3 characters"
-            + " maxLength=2 characters",
-        () -> validateAndParseString("COL", "abc", Optional.of(2)));
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type STRING, Row Index: 0, reason:"
+            + " String too long: length=3 characters maxLength=2 characters",
+        () -> validateAndParseString("COL", "abc", Optional.of(2), 0));
 
     // BINARY
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type BINARY. Allowed Java types:"
-            + " byte[], String",
-        () -> validateAndParseBinary("COL", new Object(), Optional.empty()));
+            + " cannot be ingested into Snowflake column COL of type BINARY, Row Index:0. Allowed"
+            + " Java types: byte[], String",
+        () -> validateAndParseBinary("COL", new Object(), Optional.empty(), 0));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: byte[2]. Value cannot be"
-            + " ingested into Snowflake column COL of type BINARY: Binary too long: length=2"
-            + " maxLength=1",
-        () -> validateAndParseBinary("COL", new byte[] {1, 2}, Optional.of(1)));
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type BINARY, Row Index: 0, reason:"
+            + " Binary too long: length=2 maxLength=1",
+        () -> validateAndParseBinary("COL", new byte[] {1, 2}, Optional.of(1), 0));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: ghi. Value cannot be ingested"
-            + " into Snowflake column COL of type BINARY: Not a valid hex string",
-        () -> validateAndParseBinary("COL", "ghi", Optional.empty()));
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type BINARY, Row Index: 0, reason:"
+            + " Not a valid hex string",
+        () -> validateAndParseBinary("COL", "ghi", Optional.empty(), 0));
 
     // VARIANT
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type VARIANT. Allowed Java types:"
-            + " String, Primitive data types and their arrays, java.time.*, List<T>, Map<String,"
-            + " T>, T[]",
-        () -> validateAndParseVariant("COL", new Object()));
+            + " cannot be ingested into Snowflake column COL of type VARIANT, Row Index:0. Allowed"
+            + " Java types: String, Primitive data types and their arrays, java.time.*, List<T>,"
+            + " Map<String, T>, T[]",
+        () -> validateAndParseVariant("COL", new Object(), 0));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: ][. Value cannot be ingested"
-            + " into Snowflake column COL of type VARIANT: Not a valid JSON",
-        () -> validateAndParseVariant("COL", "]["));
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type VARIANT, Row Index: 0, reason:"
+            + " Not a valid JSON",
+        () -> validateAndParseVariant("COL", "][", 0));
 
     // ARRAY
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type ARRAY. Allowed Java types:"
-            + " String, Primitive data types and their arrays, java.time.*, List<T>, Map<String,"
-            + " T>, T[]",
-        () -> validateAndParseArray("COL", new Object()));
+            + " cannot be ingested into Snowflake column COL of type ARRAY, Row Index:0. Allowed"
+            + " Java types: String, Primitive data types and their arrays, java.time.*, List<T>,"
+            + " Map<String, T>, T[]",
+        () -> validateAndParseArray("COL", new Object(), 0));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: ][. Value cannot be ingested"
-            + " into Snowflake column COL of type ARRAY: Not a valid JSON",
-        () -> validateAndParseArray("COL", "]["));
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type ARRAY, Row Index: 0, reason:"
+            + " Not a valid JSON",
+        () -> validateAndParseArray("COL", "][", 0));
 
     // OBJECT
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
+        ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type OBJECT. Allowed Java types:"
-            + " String, Primitive data types and their arrays, java.time.*, List<T>, Map<String,"
-            + " T>, T[]",
-        () -> validateAndParseObject("COL", new Object()));
+            + " cannot be ingested into Snowflake column COL of type OBJECT, Row Index:0. Allowed"
+            + " Java types: String, Primitive data types and their arrays, java.time.*, List<T>,"
+            + " Map<String, T>, T[]",
+        () -> validateAndParseObject("COL", new Object(), 0));
     expectErrorCodeAndMessage(
-        ErrorCode.INVALID_ROW,
-        "The given row cannot be converted to the internal format: }{. Value cannot be ingested"
-            + " into Snowflake column COL of type OBJECT: Not a valid JSON",
-        () -> validateAndParseObject("COL", "}{"));
+        ErrorCode.INVALID_VALUE_ROW,
+        "The given row cannot be converted to the internal format due to invalid value: Value"
+            + " cannot be ingested into Snowflake column COL of type OBJECT, Row Index: 0, reason:"
+            + " Not a valid JSON",
+        () -> validateAndParseObject("COL", "}{", 0));
   }
 
   private JsonNode readTree(String value) {
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
index 71237f5f9..6878478e2 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParquetValueParserTest.java
@@ -31,7 +31,7 @@ public void parseValueFixedSB1ToInt32() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            12, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats, UTC);
+            12, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats, UTC, 0);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -57,7 +57,7 @@ public void parseValueFixedSB2ToInt32() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            1234, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats, UTC);
+            1234, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats, UTC, 0);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -83,7 +83,7 @@ public void parseValueFixedSB4ToInt32() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            123456789, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats, UTC);
+            123456789, testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats, UTC, 0);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -113,7 +113,8 @@ public void parseValueFixedSB8ToInt64() {
             testCol,
             PrimitiveType.PrimitiveTypeName.INT64,
             rowBufferStats,
-            UTC);
+            UTC,
+            0);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -143,7 +144,8 @@ public void parseValueFixedSB16ToByteArray() {
             testCol,
             PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
             rowBufferStats,
-            UTC);
+            UTC,
+            0);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -175,7 +177,8 @@ public void parseValueFixedDecimalToInt32() {
             testCol,
             PrimitiveType.PrimitiveTypeName.DOUBLE,
             rowBufferStats,
-            UTC);
+            UTC,
+            0);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -199,7 +202,7 @@ public void parseValueDouble() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            12345.54321d, testCol, PrimitiveType.PrimitiveTypeName.DOUBLE, rowBufferStats, UTC);
+            12345.54321d, testCol, PrimitiveType.PrimitiveTypeName.DOUBLE, rowBufferStats, UTC, 0);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -223,7 +226,7 @@ public void parseValueBoolean() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            true, testCol, PrimitiveType.PrimitiveTypeName.BOOLEAN, rowBufferStats, UTC);
+            true, testCol, PrimitiveType.PrimitiveTypeName.BOOLEAN, rowBufferStats, UTC, 0);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -251,7 +254,8 @@ public void parseValueBinary() {
             testCol,
             PrimitiveType.PrimitiveTypeName.BINARY,
             rowBufferStats,
-            UTC);
+            UTC,
+            0);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -288,7 +292,7 @@ private void testJsonWithLogicalType(String logicalType) {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            var, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats, UTC);
+            var, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats, UTC, 0);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -329,7 +333,7 @@ private void testNullJsonWithLogicalType(String var) {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            var, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats, UTC);
+            var, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats, UTC, 0);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -359,7 +363,7 @@ public void parseValueArrayToBinary() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            input, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats, UTC);
+            input, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats, UTC, 0);
 
     String resultArray = "[{\"a\":\"1\",\"b\":\"2\",\"c\":\"3\"}]";
 
@@ -391,7 +395,7 @@ public void parseValueTextToBinary() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            text, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats, UTC);
+            text, testCol, PrimitiveType.PrimitiveTypeName.BINARY, rowBufferStats, UTC, 0);
 
     String result = text;
 
@@ -429,7 +433,8 @@ public void parseValueTimestampNtzSB4Error() {
                     testCol,
                     PrimitiveType.PrimitiveTypeName.INT32,
                     rowBufferStats,
-                    UTC));
+                    UTC,
+                    0));
     Assert.assertEquals(
         "Unknown data type for logical: TIMESTAMP_NTZ, physical: SB4.", exception.getMessage());
   }
@@ -452,7 +457,8 @@ public void parseValueTimestampNtzSB8ToINT64() {
             testCol,
             PrimitiveType.PrimitiveTypeName.INT64,
             rowBufferStats,
-            UTC);
+            UTC,
+            0);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -481,7 +487,8 @@ public void parseValueTimestampNtzSB16ToByteArray() {
             testCol,
             PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
             rowBufferStats,
-            UTC);
+            UTC,
+            0);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -507,7 +514,7 @@ public void parseValueDateToInt32() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            "2021-01-01", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats, UTC);
+            "2021-01-01", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats, UTC, 0);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -532,7 +539,7 @@ public void parseValueTimeSB4ToInt32() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            "01:00:00", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats, UTC);
+            "01:00:00", testCol, PrimitiveType.PrimitiveTypeName.INT32, rowBufferStats, UTC, 0);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -557,7 +564,7 @@ public void parseValueTimeSB8ToInt64() {
     RowBufferStats rowBufferStats = new RowBufferStats("COL1");
     ParquetValueParser.ParquetBufferValue pv =
         ParquetValueParser.parseColumnValueToParquet(
-            "01:00:00.123", testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats, UTC);
+            "01:00:00.123", testCol, PrimitiveType.PrimitiveTypeName.INT64, rowBufferStats, UTC, 0);
 
     ParquetValueParserAssertionBuilder.newBuilder()
         .parquetBufferValue(pv)
@@ -589,7 +596,8 @@ public void parseValueTimeSB16Error() {
                     testCol,
                     PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY,
                     rowBufferStats,
-                    UTC));
+                    UTC,
+                    0));
     Assert.assertEquals(
         "Unknown data type for logical: TIME, physical: SB16.", exception.getMessage());
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 16b8b67d5..aa6606745 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -330,14 +330,14 @@ private void testStringLengthHelper(AbstractRowBuffer<?> rowBuffer) {
       Assert.assertTrue(response.hasErrors());
       Assert.assertEquals(1, response.getErrorRowCount());
       Assert.assertEquals(
-          ErrorCode.INVALID_ROW.getMessageCode(),
+          ErrorCode.INVALID_VALUE_ROW.getMessageCode(),
           response.getInsertErrors().get(0).getException().getVendorCode());
       Assert.assertTrue(response.getInsertErrors().get(0).getMessage().contains("String too long"));
     } else {
       try {
         rowBuffer.insertRows(Collections.singletonList(row), null);
       } catch (SFException e) {
-        Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
+        Assert.assertEquals(ErrorCode.INVALID_VALUE_ROW.getMessageCode(), e.getVendorCode());
       }
     }
   }
@@ -657,13 +657,13 @@ private void testE2ETimestampErrorsHelper(AbstractRowBuffer<?> innerBuffer) {
           innerBuffer.insertRows(Collections.singletonList(row), null);
       Assert.assertTrue(response.hasErrors());
       Assert.assertEquals(
-          ErrorCode.INVALID_ROW.getMessageCode(),
+          ErrorCode.INVALID_FORMAT_ROW.getMessageCode(),
           response.getInsertErrors().get(0).getException().getVendorCode());
     } else {
       try {
         innerBuffer.insertRows(Collections.singletonList(row), null);
       } catch (SFException e) {
-        Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
+        Assert.assertEquals(ErrorCode.INVALID_FORMAT_ROW.getMessageCode(), e.getVendorCode());
       }
     }
   }
@@ -979,13 +979,13 @@ private void testNullableCheckHelper(OpenChannelRequest.OnErrorOption onErrorOpt
       response = innerBuffer.insertRows(Collections.singletonList(row), "1");
       Assert.assertTrue(response.hasErrors());
       Assert.assertEquals(
-          ErrorCode.INVALID_ROW.getMessageCode(),
+          ErrorCode.INVALID_FORMAT_ROW.getMessageCode(),
           response.getInsertErrors().get(0).getException().getVendorCode());
     } else {
       try {
         innerBuffer.insertRows(Collections.singletonList(row), "1");
       } catch (SFException e) {
-        Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
+        Assert.assertEquals(ErrorCode.INVALID_FORMAT_ROW.getMessageCode(), e.getVendorCode());
       }
     }
   }
@@ -1027,14 +1027,14 @@ private void testMissingColumnCheckHelper(OpenChannelRequest.OnErrorOption onErr
       Assert.assertTrue(response.hasErrors());
       InsertValidationResponse.InsertError error = response.getInsertErrors().get(0);
       Assert.assertEquals(
-          ErrorCode.INVALID_ROW.getMessageCode(), error.getException().getVendorCode());
+          ErrorCode.INVALID_FORMAT_ROW.getMessageCode(), error.getException().getVendorCode());
       Assert.assertEquals(
           Collections.singletonList("COLBOOLEAN"), error.getMissingNotNullColNames());
     } else {
       try {
         innerBuffer.insertRows(Collections.singletonList(row2), "2");
       } catch (SFException e) {
-        Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
+        Assert.assertEquals(ErrorCode.INVALID_FORMAT_ROW.getMessageCode(), e.getVendorCode());
       }
     }
   }
@@ -1060,7 +1060,7 @@ public void testExtraColumnsCheck() {
     Assert.assertTrue(response.hasErrors());
     InsertValidationResponse.InsertError error = response.getInsertErrors().get(0);
     Assert.assertEquals(
-        ErrorCode.INVALID_ROW.getMessageCode(), error.getException().getVendorCode());
+        ErrorCode.INVALID_FORMAT_ROW.getMessageCode(), error.getException().getVendorCode());
     Assert.assertEquals(Arrays.asList("COLBOOLEAN3", "COLBOOLEAN2"), error.getExtraColNames());
   }
 
@@ -1314,7 +1314,7 @@ public void testOnErrorAbortFailures() {
     InsertValidationResponse response = innerBuffer.insertRows(Collections.singletonList(row), "1");
     Assert.assertFalse(response.hasErrors());
 
-    Assert.assertEquals(1, innerBuffer.rowCount);
+    Assert.assertEquals(1, innerBuffer.bufferedRowCount);
     Assert.assertEquals(0, innerBuffer.getTempRowCount());
     Assert.assertEquals(
         1, innerBuffer.statsMap.get("COLDECIMAL").getCurrentMaxIntValue().intValue());
@@ -1328,7 +1328,7 @@ public void testOnErrorAbortFailures() {
     response = innerBuffer.insertRows(Collections.singletonList(row2), "2");
     Assert.assertFalse(response.hasErrors());
 
-    Assert.assertEquals(2, innerBuffer.rowCount);
+    Assert.assertEquals(2, innerBuffer.bufferedRowCount);
     Assert.assertEquals(0, innerBuffer.getTempRowCount());
     Assert.assertEquals(
         2, innerBuffer.statsMap.get("COLDECIMAL").getCurrentMaxIntValue().intValue());
@@ -1342,10 +1342,10 @@ public void testOnErrorAbortFailures() {
     try {
       innerBuffer.insertRows(Collections.singletonList(row3), "3");
     } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
+      Assert.assertEquals(ErrorCode.INVALID_FORMAT_ROW.getMessageCode(), e.getVendorCode());
     }
 
-    Assert.assertEquals(2, innerBuffer.rowCount);
+    Assert.assertEquals(2, innerBuffer.bufferedRowCount);
     Assert.assertEquals(0, innerBuffer.getTempRowCount());
     Assert.assertEquals(
         2, innerBuffer.statsMap.get("COLDECIMAL").getCurrentMaxIntValue().intValue());
@@ -1357,7 +1357,7 @@ public void testOnErrorAbortFailures() {
     row3.put("COLDECIMAL", 3);
     response = innerBuffer.insertRows(Collections.singletonList(row3), "3");
     Assert.assertFalse(response.hasErrors());
-    Assert.assertEquals(3, innerBuffer.rowCount);
+    Assert.assertEquals(3, innerBuffer.bufferedRowCount);
     Assert.assertEquals(0, innerBuffer.getTempRowCount());
     Assert.assertEquals(
         3, innerBuffer.statsMap.get("COLDECIMAL").getCurrentMaxIntValue().intValue());
@@ -1368,7 +1368,7 @@ public void testOnErrorAbortFailures() {
 
     ChannelData<?> data = innerBuffer.flush("my_snowpipe_streaming.bdec");
     Assert.assertEquals(3, data.getRowCount());
-    Assert.assertEquals(0, innerBuffer.rowCount);
+    Assert.assertEquals(0, innerBuffer.bufferedRowCount);
   }
 
   @Test
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index e302e3610..3ff33e43b 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -841,13 +841,13 @@ public void testAbortOnErrorOption() throws Exception {
       channel.insertRow(row3, "3");
       Assert.fail("insert should fail");
     } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
+      Assert.assertEquals(ErrorCode.INVALID_VALUE_ROW.getMessageCode(), e.getVendorCode());
     }
     try {
       channel.insertRows(Arrays.asList(row1, row2, row3), "6");
       Assert.fail("insert should fail");
     } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.INVALID_ROW.getMessageCode(), e.getVendorCode());
+      Assert.assertEquals(ErrorCode.INVALID_VALUE_ROW.getMessageCode(), e.getVendorCode());
     }
     Map<String, Object> row7 = new HashMap<>();
     row7.put("c1", 7);

From a7e39bf33068c84e0e48345bb0ca01fe50de31fc Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 12 Apr 2023 14:16:31 -0700
Subject: [PATCH 183/356] SNOW-787565: Use dynamic scaling thread pool instead
 of fixed thread pool (#470)

Use dynamic scaling thread pool instead of fixed thread pool
---
 .../ingest/streaming/internal/FlushService.java          | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index a01f82e85..7ab1fd1c5 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -31,6 +31,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -320,7 +321,13 @@ private void createWorkers() {
                 * (1 + this.owningClient.getParameterProvider().getIOTimeCpuRatio()),
             MAX_THREAD_COUNT);
     this.buildUploadWorkers =
-        Executors.newFixedThreadPool(buildUploadThreadCount, buildUploadThreadFactory);
+        new ThreadPoolExecutor(
+            1,
+            buildUploadThreadCount,
+            60L,
+            TimeUnit.SECONDS,
+            new SynchronousQueue<Runnable>(),
+            buildUploadThreadFactory);
 
     logger.logInfo(
         "Create {} threads for build/upload blobs for client={}, total available processors={}",

From cf880ce72b0752be1eb3778b343b7755feaa8aa8 Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Fri, 14 Apr 2023 16:14:40 +0200
Subject: [PATCH 184/356] @no-snow fix bug in OnErrorOption.Abort for Parquet
 (#452)

---
 .../streaming/internal/ParquetRowBuffer.java  |  2 +-
 .../streaming/internal/RowBufferTest.java     | 80 +++++++++++++++++++
 2 files changed, 81 insertions(+), 1 deletion(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 1425a5e9c..d56f30b69 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -164,7 +164,7 @@ float addTempRow(
       Set<String> formattedInputColumnNames,
       long insertRowIndex) {
     return addRow(
-        row, curRowIndex, this::writeRow, statsMap, formattedInputColumnNames, insertRowIndex);
+        row, curRowIndex, tempData::add, statsMap, formattedInputColumnNames, insertRowIndex);
   }
 
   /**
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index aa6606745..170619d05 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -19,7 +19,9 @@
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.lang3.NotImplementedException;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -1498,4 +1500,82 @@ private void testE2EArrayHelper(OpenChannelRequest.OnErrorOption onErrorOption)
     ChannelData<?> result = innerBuffer.flush("my_snowpipe_streaming.bdec");
     Assert.assertEquals(5, result.getRowCount());
   }
+
+  @Test
+  public void testOnErrorAbortRowsWithError() {
+    AbstractRowBuffer<?> innerBufferOnErrorContinue =
+        createTestBuffer(OpenChannelRequest.OnErrorOption.CONTINUE);
+    AbstractRowBuffer<?> innerBufferOnErrorAbort =
+        createTestBuffer(OpenChannelRequest.OnErrorOption.ABORT);
+
+    ColumnMetadata colChar = new ColumnMetadata();
+    colChar.setName("COLCHAR");
+    colChar.setPhysicalType("LOB");
+    colChar.setNullable(true);
+    colChar.setLogicalType("TEXT");
+    colChar.setByteLength(14);
+    colChar.setLength(11);
+    colChar.setScale(0);
+
+    innerBufferOnErrorContinue.setupSchema(Collections.singletonList(colChar));
+    innerBufferOnErrorAbort.setupSchema(Collections.singletonList(colChar));
+
+    // insert one valid row
+    List<Map<String, Object>> validRows = new ArrayList<>();
+    validRows.add(Collections.singletonMap("colChar", "a"));
+
+    InsertValidationResponse response = innerBufferOnErrorContinue.insertRows(validRows, "1");
+    Assert.assertFalse(response.hasErrors());
+    response = innerBufferOnErrorAbort.insertRows(validRows, "1");
+    Assert.assertFalse(response.hasErrors());
+
+    // insert one valid and one invalid row
+    List<Map<String, Object>> mixedRows = new ArrayList<>();
+    mixedRows.add(Collections.singletonMap("colChar", "b"));
+    mixedRows.add(Collections.singletonMap("colChar", "1111111111111111111111")); // too big
+
+    response = innerBufferOnErrorContinue.insertRows(mixedRows, "3");
+    Assert.assertTrue(response.hasErrors());
+
+    Assert.assertThrows(
+        SFException.class, () -> innerBufferOnErrorAbort.insertRows(mixedRows, "3"));
+
+    switch (bdecVersion) {
+      case ONE:
+        VectorSchemaRoot snapshotContinueArrow =
+            ((VectorSchemaRoot) innerBufferOnErrorContinue.getSnapshot("fake/filePath").get());
+        // validRows and only the good row from mixedRows are in the buffer
+        Assert.assertEquals(2, snapshotContinueArrow.getRowCount());
+        Assert.assertEquals("[a, b]", snapshotContinueArrow.getVector(0).toString());
+
+        VectorSchemaRoot snapshotAbortArrow =
+            ((VectorSchemaRoot) innerBufferOnErrorAbort.getSnapshot("fake/filePath").get());
+        // only validRows and none of the mixedRows are in the buffer
+        Assert.assertEquals(1, snapshotAbortArrow.getRowCount());
+        Assert.assertEquals("[a]", snapshotAbortArrow.getVector(0).toString());
+        break;
+
+      case THREE:
+        List<List<Object>> snapshotContinueParquet =
+            ((ParquetChunkData) innerBufferOnErrorContinue.getSnapshot("fake/filePath").get()).rows;
+        // validRows and only the good row from mixedRows are in the buffer
+        Assert.assertEquals(2, snapshotContinueParquet.size());
+        Assert.assertEquals(Arrays.asList("a"), snapshotContinueParquet.get(0));
+        Assert.assertEquals(Arrays.asList("b"), snapshotContinueParquet.get(1));
+
+        List<List<Object>> snapshotAbortParquet =
+            ((ParquetChunkData) innerBufferOnErrorAbort.getSnapshot("fake/filePath").get()).rows;
+        // only validRows and none of the mixedRows are in the buffer
+        Assert.assertEquals(1, snapshotAbortParquet.size());
+        Assert.assertEquals(Arrays.asList("a"), snapshotAbortParquet.get(0));
+        break;
+      default:
+        throw new NotImplementedException("Unsupported version!");
+    }
+    if (bdecVersion == Constants.BdecVersion.THREE) {
+
+    } else if (bdecVersion == Constants.BdecVersion.ONE) {
+
+    }
+  }
 }

From 6ae14209641282e6c05c265dfdf0499e1ef390dc Mon Sep 17 00:00:00 2001
From: Jay Patel <jay.patel@snowflake.com>
Date: Fri, 14 Apr 2023 11:10:58 -0700
Subject: [PATCH 185/356] SNOW-726924 Re-add jvmNonProxy host (#474)

* Revert "NO-SNOW: revert https://github.com/snowflakedb/snowflake-kafka-connector/pull/546/files as it's causing failure on GCS (#334)"

This reverts commit ca63c27621f2f143638bde113fcfae4a400d145f.

* Upgrade JDBC to 3.13.25

* Upgrade maven-shade-plugin to recent version to support java 11

Failed to execute goal org.apache.maven.plugins:maven-shade-plugin:3.1.1:shade (default) on project snowflake-ingest-sdk: Error creating shaded jar: Problem shading JAR /Users/japatel/.m2/repository/net/snowflake/snowflake-jdbc/3.13.25/snowflake-jdbc-3.13.25.jar entry META-INF/versions/11/org/bouncycastle/jcajce/provider/asymmetric/edec/KeyAgreementSpi$X448.class: java.lang.IllegalArgumentException -> [Help 1]
---
 pom.xml                                       |  4 +-
 .../snowflake/ingest/SimpleIngestManager.java |  2 +-
 ...nowflakeStreamingIngestClientInternal.java |  5 +-
 .../net/snowflake/ingest/utils/HttpUtil.java  | 49 +++++++++++++++++--
 .../internal/StreamingIngestStageTest.java    | 45 +++++++++++++++++
 5 files changed, 96 insertions(+), 9 deletions(-)

diff --git a/pom.xml b/pom.xml
index 9c74d467d..f1fe1e82a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -60,7 +60,7 @@
     <shadeBase>net.snowflake.ingest.internal</shadeBase>
     <slf4j.version>1.7.36</slf4j.version>
     <snappy.version>1.1.8.3</snappy.version>
-    <snowjdbc.version>3.13.15</snowjdbc.version>
+    <snowjdbc.version>3.13.25</snowjdbc.version>
     <yetus.version>0.13.0</yetus.version>
   </properties>
 
@@ -701,7 +701,7 @@
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-shade-plugin</artifactId>
-            <version>3.1.1</version>
+            <version>3.4.1</version>
             <configuration>
               <relocations>
                 <relocation>
diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
index e5163a4dc..d167488dd 100644
--- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
+++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
@@ -407,7 +407,7 @@ private void init(String account, String user, String pipe, KeyPair keyPair) {
     this.keyPair = keyPair;
 
     // make our client for sending requests
-    httpClient = HttpUtil.getHttpClient();
+    httpClient = HttpUtil.getHttpClient(account);
     // make the request builder we'll use to build messages to the service
   }
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index dbbcb6562..5ea65108b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -102,6 +102,8 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
   // Name of the client
   private final String name;
 
+  private String accountName;
+
   // Snowflake role for the client to use
   private String role;
 
@@ -166,8 +168,9 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
     this.parameterProvider = new ParameterProvider(parameterOverrides, prop);
 
     this.name = name;
+    this.accountName = accountURL == null ? null : accountURL.getAccount();
     this.isTestMode = isTestMode;
-    this.httpClient = httpClient == null ? HttpUtil.getHttpClient() : httpClient;
+    this.httpClient = httpClient == null ? HttpUtil.getHttpClient(accountName) : httpClient;
     this.channelCache = new ChannelCache<>();
     this.allocator = new RootAllocator();
     this.isClosed = false;
diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index 3b34459f5..72bf32767 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -14,6 +14,7 @@
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
 import javax.net.ssl.SSLContext;
 import net.snowflake.client.core.SFSessionProperty;
 import net.snowflake.client.jdbc.internal.apache.http.HttpHost;
@@ -47,9 +48,10 @@ public class HttpUtil {
   public static final String USE_PROXY = "http.useProxy";
   public static final String PROXY_HOST = "http.proxyHost";
   public static final String PROXY_PORT = "http.proxyPort";
-
+  public static final String NON_PROXY_HOSTS = "http.nonProxyHosts";
   public static final String HTTP_PROXY_USER = "http.proxyUser";
   public static final String HTTP_PROXY_PASSWORD = "http.proxyPassword";
+  private static final String SNOWFLAKE_DOMAIN_NAME = ".snowflakecomputing.com";
 
   private static final String PROXY_SCHEME = "http";
   private static final String FIRST_FAULT_TIMESTAMP = "FIRST_FAULT_TIMESTAMP";
@@ -84,11 +86,15 @@ public class HttpUtil {
   // Only connections that are currently owned, not checked out, are subject to idle timeouts.
   private static final int DEFAULT_IDLE_CONNECTION_TIMEOUT_SECONDS = 30;
 
-  public static CloseableHttpClient getHttpClient() {
+  /**
+   * @param {@code String} account name to connect to (excluding snowflakecomputing.com domain)
+   * @return Instance of CloseableHttpClient
+   */
+  public static CloseableHttpClient getHttpClient(String accountName) {
     if (httpClient == null) {
       synchronized (HttpUtil.class) {
         if (httpClient == null) {
-          initHttpClient();
+          initHttpClient(accountName);
         }
       }
     }
@@ -100,7 +106,7 @@ public static CloseableHttpClient getHttpClient() {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(HttpUtil.class);
 
-  private static void initHttpClient() {
+  private static void initHttpClient(String accountName) {
 
     Security.setProperty("ocsp.enable", "true");
 
@@ -146,7 +152,7 @@ private static void initHttpClient() {
             .setDefaultRequestConfig(requestConfig);
 
     // proxy settings
-    if ("true".equalsIgnoreCase(System.getProperty(USE_PROXY))) {
+    if ("true".equalsIgnoreCase(System.getProperty(USE_PROXY)) && !shouldBypassProxy(accountName)) {
       if (System.getProperty(PROXY_PORT) == null) {
         throw new IllegalArgumentException(
             "proxy port number is not provided, please assign proxy port to http.proxyPort option");
@@ -316,6 +322,12 @@ public static Properties generateProxyPropertiesForJDBC() {
         proxyProperties.put(SFSessionProperty.PROXY_USER.getPropertyKey(), proxyUser);
         proxyProperties.put(SFSessionProperty.PROXY_PASSWORD.getPropertyKey(), proxyPassword);
       }
+
+      // Check if http.nonProxyHosts was set
+      final String nonProxyHosts = System.getProperty(NON_PROXY_HOSTS);
+      if (!isNullOrEmpty(nonProxyHosts)) {
+        proxyProperties.put(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey(), nonProxyHosts);
+      }
     }
     return proxyProperties;
   }
@@ -401,4 +413,31 @@ private static String createPoolStatsInfo(String title, PoolStats poolStats) {
     }
     return title;
   }
+
+  /**
+   * Changes the account name to the format accountName.snowflakecomputing.com then returns a
+   * boolean to indicate if we should go through a proxy or not.
+   */
+  public static Boolean shouldBypassProxy(String accountName) {
+    String targetHost = accountName + SNOWFLAKE_DOMAIN_NAME;
+    return System.getProperty(NON_PROXY_HOSTS) != null && isInNonProxyHosts(targetHost);
+  }
+
+  /**
+   * The target hostname input is compared with the hosts in the '|' separated list provided by the
+   * http.nonProxyHosts parameter using regex. The nonProxyHosts will be used as our Patterns, so we
+   * need to replace the '.' and '*' characters since those are special regex constructs that mean
+   * 'any character,' and 'repeat 0 or more times.'
+   */
+  private static Boolean isInNonProxyHosts(String targetHost) {
+    String nonProxyHosts =
+        System.getProperty(NON_PROXY_HOSTS).replace(".", "\\.").replace("*", ".*");
+    String[] nonProxyHostsArray = nonProxyHosts.split("\\|");
+    for (String i : nonProxyHostsArray) {
+      if (Pattern.compile(i).matcher(targetHost).matches()) {
+        return true;
+      }
+    }
+    return false;
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
index 456154982..bdbe42df8 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
@@ -3,10 +3,12 @@
 import static net.snowflake.client.core.Constants.CLOUD_STORAGE_CREDENTIALS_EXPIRED;
 import static net.snowflake.ingest.utils.HttpUtil.HTTP_PROXY_PASSWORD;
 import static net.snowflake.ingest.utils.HttpUtil.HTTP_PROXY_USER;
+import static net.snowflake.ingest.utils.HttpUtil.NON_PROXY_HOSTS;
 import static net.snowflake.ingest.utils.HttpUtil.PROXY_HOST;
 import static net.snowflake.ingest.utils.HttpUtil.PROXY_PORT;
 import static net.snowflake.ingest.utils.HttpUtil.USE_PROXY;
 import static net.snowflake.ingest.utils.HttpUtil.generateProxyPropertiesForJDBC;
+import static net.snowflake.ingest.utils.HttpUtil.shouldBypassProxy;
 import static org.mockito.Mockito.times;
 
 import java.io.ByteArrayInputStream;
@@ -394,11 +396,13 @@ public void testGenerateProxyPropertiesForJDBC() {
     String oldProxyPort = System.getProperty(PROXY_PORT);
     String oldUser = System.getProperty(HTTP_PROXY_USER);
     String oldPassword = System.getProperty(HTTP_PROXY_PASSWORD);
+    String oldNonProxyHosts = System.getProperty(NON_PROXY_HOSTS);
 
     String proxyHost = "localhost";
     String proxyPort = "8080";
     String user = "admin";
     String password = "test";
+    String nonProxyHosts = "*.snowflakecomputing.com";
 
     try {
       // Test empty properties when USE_PROXY is NOT set;
@@ -410,6 +414,7 @@ public void testGenerateProxyPropertiesForJDBC() {
       System.setProperty(PROXY_PORT, proxyPort);
       System.setProperty(HTTP_PROXY_USER, user);
       System.setProperty(HTTP_PROXY_PASSWORD, password);
+      System.setProperty(NON_PROXY_HOSTS, nonProxyHosts);
 
       // Verify that properties are set
       props = generateProxyPropertiesForJDBC();
@@ -418,6 +423,8 @@ public void testGenerateProxyPropertiesForJDBC() {
       Assert.assertEquals(proxyPort, props.get(SFSessionProperty.PROXY_PORT.getPropertyKey()));
       Assert.assertEquals(user, props.get(SFSessionProperty.PROXY_USER.getPropertyKey()));
       Assert.assertEquals(password, props.get(SFSessionProperty.PROXY_PASSWORD.getPropertyKey()));
+      Assert.assertEquals(
+          nonProxyHosts, props.get(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey()));
     } finally {
       // Cleanup
       if (oldUseProxy != null) {
@@ -429,6 +436,44 @@ public void testGenerateProxyPropertiesForJDBC() {
         System.setProperty(HTTP_PROXY_USER, oldUser);
         System.setProperty(HTTP_PROXY_PASSWORD, oldPassword);
       }
+      if (oldNonProxyHosts != null) {
+        System.setProperty(NON_PROXY_HOSTS, oldNonProxyHosts);
+      }
+    }
+  }
+
+  @Test
+  public void testShouldBypassProxy() {
+    String oldNonProxyHosts = System.getProperty(NON_PROXY_HOSTS);
+    String accountName = "accountName12345";
+    String accountDashName = "account-name12345";
+    String accountUnderscoreName = "account_name12345";
+    String nonProxyHosts = "*.snowflakecomputing.com|localhost";
+
+    System.setProperty(NON_PROXY_HOSTS, nonProxyHosts);
+    Assert.assertTrue(shouldBypassProxy(accountName));
+    Assert.assertTrue(shouldBypassProxy(accountDashName));
+    Assert.assertTrue(shouldBypassProxy(accountUnderscoreName));
+
+    String accountNamePrivateLink = "accountName12345.privatelink";
+    nonProxyHosts = "*.privatelink.snowflakecomputing.com|localhost";
+    System.setProperty(NON_PROXY_HOSTS, nonProxyHosts);
+    Assert.assertTrue(shouldBypassProxy(accountNamePrivateLink));
+
+    // Previous tests should return false with the new nonProxyHosts value
+    Assert.assertFalse(shouldBypassProxy(accountName));
+    Assert.assertFalse(shouldBypassProxy(accountDashName));
+    Assert.assertFalse(shouldBypassProxy(accountUnderscoreName));
+
+    // All tests should return false after clearing the nonProxyHosts property
+    System.clearProperty(NON_PROXY_HOSTS);
+    Assert.assertFalse(shouldBypassProxy(accountName));
+    Assert.assertFalse(shouldBypassProxy(accountDashName));
+    Assert.assertFalse(shouldBypassProxy(accountUnderscoreName));
+    Assert.assertFalse(shouldBypassProxy(accountNamePrivateLink));
+
+    if (oldNonProxyHosts != null) {
+      System.setProperty(NON_PROXY_HOSTS, oldNonProxyHosts);
     }
   }
 }

From 7106abd1dc6cbdb90c348fa6f6a81391f6b5c8a2 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Fri, 14 Apr 2023 12:55:36 -0700
Subject: [PATCH 186/356] NO-SNOW: Preserve old JDBC behavior after upgrade
 (#475)

Follow-up PR to preserve the old JDBC behavior for understore account name after #474
---
 ...SnowflakeStreamingIngestClientInternal.java |  5 +++--
 .../net/snowflake/ingest/utils/Constants.java  |  1 -
 .../java/net/snowflake/ingest/utils/Utils.java | 18 ++++++++++++++----
 .../SnowflakeStreamingIngestChannelTest.java   |  6 ++++--
 .../SnowflakeStreamingIngestClientTest.java    |  6 ++++--
 5 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 5ea65108b..523c71841 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -13,7 +13,6 @@
 import static net.snowflake.ingest.utils.Constants.COMMIT_MAX_RETRY_COUNT;
 import static net.snowflake.ingest.utils.Constants.COMMIT_RETRY_INTERVAL_IN_MS;
 import static net.snowflake.ingest.utils.Constants.ENABLE_TELEMETRY_TO_SF;
-import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
 import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
 import static net.snowflake.ingest.utils.Constants.OPEN_CHANNEL_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
@@ -58,6 +57,7 @@
 import java.util.stream.Collectors;
 import javax.management.MalformedObjectNameException;
 import javax.management.ObjectName;
+import net.snowflake.client.core.SFSessionProperty;
 import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.ingest.connection.IngestResponseException;
 import net.snowflake.ingest.connection.RequestBuilder;
@@ -181,7 +181,8 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
       this.role = prop.getProperty(Constants.ROLE);
       try {
         KeyPair keyPair =
-            Utils.createKeyPairFromPrivateKey((PrivateKey) prop.get(JDBC_PRIVATE_KEY));
+            Utils.createKeyPairFromPrivateKey(
+                (PrivateKey) prop.get(SFSessionProperty.PRIVATE_KEY.getPropertyKey()));
         this.requestBuilder =
             new RequestBuilder(
                 accountURL,
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index e43c419c8..e4e399978 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -24,7 +24,6 @@ public class Constants {
   public static final String ROLE = "role";
   public static final String PRIVATE_KEY = "private_key";
   public static final String PRIVATE_KEY_PASSPHRASE = "private_key_passphrase";
-  public static final String JDBC_PRIVATE_KEY = "privateKey";
   public static final String PRIMARY_FILE_ID_KEY =
       "primaryFileId"; // Don't change, should match Parquet Scanner
   public static final long RESPONSE_SUCCESS = 0L; // Don't change, should match server side
diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java
index 2c9dbb156..1280dbf46 100644
--- a/src/main/java/net/snowflake/ingest/utils/Utils.java
+++ b/src/main/java/net/snowflake/ingest/utils/Utils.java
@@ -4,7 +4,6 @@
 
 package net.snowflake.ingest.utils;
 
-import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
 import static net.snowflake.ingest.utils.Constants.USER;
 
 import com.codahale.metrics.Timer;
@@ -26,6 +25,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import net.snowflake.client.core.SFSessionProperty;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.PEMParser;
@@ -97,12 +97,14 @@ public static Properties createProperties(Properties inputProp) {
     }
 
     if (!privateKeyPassphrase.isEmpty()) {
-      properties.put(JDBC_PRIVATE_KEY, parseEncryptedPrivateKey(privateKey, privateKeyPassphrase));
+      properties.put(
+          SFSessionProperty.PRIVATE_KEY.getPropertyKey(),
+          parseEncryptedPrivateKey(privateKey, privateKeyPassphrase));
     } else if (!privateKey.isEmpty()) {
-      properties.put(JDBC_PRIVATE_KEY, parsePrivateKey(privateKey));
+      properties.put(SFSessionProperty.PRIVATE_KEY.getPropertyKey(), parsePrivateKey(privateKey));
     }
 
-    if (!properties.containsKey(JDBC_PRIVATE_KEY)) {
+    if (!properties.containsKey(SFSessionProperty.PRIVATE_KEY.getPropertyKey())) {
       throw new SFException(ErrorCode.MISSING_CONFIG, "private_key");
     }
 
@@ -133,6 +135,14 @@ public static Properties createProperties(Properties inputProp) {
       throw new SFException(ErrorCode.MISSING_CONFIG, "role");
     }
 
+    /**
+     * Behavior change in JDBC release 3.13.25
+     *
+     * @see <a href="https://community.snowflake.com/s/article/JDBC-Driver-Release-Notes">Snowflake
+     *     Documentation Release Notes </a>
+     */
+    properties.put(SFSessionProperty.ALLOW_UNDERSCORES_IN_HOST.getPropertyKey(), "true");
+
     return properties;
   }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index b810abadb..5e593db05 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -2,7 +2,6 @@
 
 import static java.time.ZoneOffset.UTC;
 import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
-import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
 import static net.snowflake.ingest.utils.Constants.OPEN_CHANNEL_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
@@ -20,6 +19,7 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import net.snowflake.client.core.SFSessionProperty;
 import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils;
 import net.snowflake.client.jdbc.internal.apache.http.HttpEntity;
 import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders;
@@ -251,7 +251,9 @@ public void testOpenChannelPostRequest() throws Exception {
     String urlStr = "https://sfctest0.snowflakecomputing.com:80";
     SnowflakeURL url = new SnowflakeURL(urlStr);
 
-    KeyPair keyPair = Utils.createKeyPairFromPrivateKey((PrivateKey) prop.get(JDBC_PRIVATE_KEY));
+    KeyPair keyPair =
+        Utils.createKeyPairFromPrivateKey(
+            (PrivateKey) prop.get(SFSessionProperty.PRIVATE_KEY.getPropertyKey()));
     RequestBuilder requestBuilder =
         new RequestBuilder(url, prop.get(USER).toString(), keyPair, null, null);
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index e799e1b7c..3b50c8354 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -3,7 +3,6 @@
 import static java.time.ZoneOffset.UTC;
 import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
 import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.JDBC_PRIVATE_KEY;
 import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
 import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
 import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
@@ -32,6 +31,7 @@
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
+import net.snowflake.client.core.SFSessionProperty;
 import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils;
 import net.snowflake.client.jdbc.internal.apache.http.HttpEntity;
 import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders;
@@ -440,7 +440,9 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
     String urlStr = "https://sfctest0.snowflakecomputing.com:80";
     SnowflakeURL url = new SnowflakeURL(urlStr);
 
-    KeyPair keyPair = Utils.createKeyPairFromPrivateKey((PrivateKey) prop.get(JDBC_PRIVATE_KEY));
+    KeyPair keyPair =
+        Utils.createKeyPairFromPrivateKey(
+            (PrivateKey) prop.get(SFSessionProperty.PRIVATE_KEY.getPropertyKey()));
     RequestBuilder requestBuilder =
         new RequestBuilder(url, prop.get(USER).toString(), keyPair, null, null);
 

From b12bbfd90dfa8f397e60aca80829958a5b7c5a3d Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Tue, 4 Apr 2023 19:07:47 +0200
Subject: [PATCH 187/356] @snow SNOW-779794 Client SDK Parquet buffer: fix perf
 degradation due to calling suboptimal lib method per row

---
 .../streaming/internal/AbstractRowBuffer.java |  2 -
 .../streaming/internal/ParquetRowBuffer.java  | 51 ++++++++++---------
 .../internal/ParquetTypeGenerator.java        |  4 ++
 ...owflakeStreamingIngestChannelInternal.java |  1 -
 .../streaming/internal/RowBufferTest.java     |  1 -
 5 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 3c2c34831..9156f9c4b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -559,7 +559,6 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelRuntimeState,
-      boolean bufferForTests,
       boolean enableParquetMemoryOptimization) {
     switch (bdecVersion) {
       case ONE:
@@ -582,7 +581,6 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
                 fullyQualifiedChannelName,
                 rowSizeMetric,
                 channelRuntimeState,
-                bufferForTests,
                 enableParquetMemoryOptimization);
       default:
         throw new SFException(
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index d56f30b69..f2db5dd10 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -22,11 +22,8 @@
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
 import net.snowflake.ingest.utils.SFException;
 import org.apache.arrow.memory.BufferAllocator;
-import org.apache.parquet.column.ColumnDescriptor;
 import org.apache.parquet.hadoop.BdecParquetWriter;
 import org.apache.parquet.schema.MessageType;
 import org.apache.parquet.schema.PrimitiveType;
@@ -37,11 +34,9 @@
  * converted to Parquet format for faster processing
  */
 public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
-  private static final Logging logger = new Logging(ParquetRowBuffer.class);
-
   private static final String PARQUET_MESSAGE_TYPE_NAME = "bdec";
 
-  private final Map<String, Pair<ColumnMetadata, Integer>> fieldIndex;
+  private final Map<String, ParquetColumn> fieldIndex;
 
   /* map that contains metadata like typeinfo for columns and other information needed by the server scanner */
   private final Map<String, String> metadata;
@@ -56,7 +51,6 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
   private final String channelName;
 
   private MessageType schema;
-  private final boolean bufferForTests;
   private final boolean enableParquetInternalBuffering;
   /** Construct a ParquetRowBuffer object. */
   ParquetRowBuffer(
@@ -66,7 +60,6 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelRuntimeState,
-      boolean bufferForTests,
       boolean enableParquetInternalBuffering) {
     super(
         onErrorOption,
@@ -80,7 +73,6 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
     data = new ArrayList<>();
     tempData = new ArrayList<>();
     channelName = fullyQualifiedChannelName;
-    this.bufferForTests = bufferForTests;
     this.enableParquetInternalBuffering = enableParquetInternalBuffering;
   }
 
@@ -97,7 +89,10 @@ public void setupSchema(List<ColumnMetadata> columns) {
           ParquetTypeGenerator.generateColumnParquetTypeInfo(column, id);
       parquetTypes.add(typeInfo.getParquetType());
       this.metadata.putAll(typeInfo.getMetadata());
-      fieldIndex.put(column.getInternalName(), new Pair<>(column, parquetTypes.size() - 1));
+      int columnIndex = parquetTypes.size() - 1;
+      fieldIndex.put(
+          column.getInternalName(),
+          new ParquetColumn(column, columnIndex, typeInfo.getPrimitiveTypeName()));
       if (!column.getNullable()) {
         addNonNullableFieldName(column.getInternalName());
       }
@@ -144,8 +139,7 @@ float addRow(
       Map<String, RowBufferStats> statsMap,
       Set<String> formattedInputColumnNames,
       final long insertRowIndex) {
-    return addRow(
-        row, bufferedRowIndex, this::writeRow, statsMap, formattedInputColumnNames, insertRowIndex);
+    return addRow(row, bufferedRowIndex, this::writeRow, statsMap, formattedInputColumnNames);
   }
 
   void writeRow(List<Object> row) {
@@ -163,8 +157,7 @@ float addTempRow(
       Map<String, RowBufferStats> statsMap,
       Set<String> formattedInputColumnNames,
       long insertRowIndex) {
-    return addRow(
-        row, curRowIndex, tempData::add, statsMap, formattedInputColumnNames, insertRowIndex);
+    return addRow(row, curRowIndex, tempData::add, statsMap, formattedInputColumnNames);
   }
 
   /**
@@ -182,8 +175,7 @@ private float addRow(
       int curRowIndex,
       Consumer<List<Object>> out,
       Map<String, RowBufferStats> statsMap,
-      Set<String> inputColumnNames,
-      final long insertRowIndex) {
+      Set<String> inputColumnNames) {
     Object[] indexedRow = new Object[fieldIndex.size()];
     float size = 0F;
 
@@ -194,16 +186,14 @@ private float addRow(
       String key = entry.getKey();
       Object value = entry.getValue();
       String columnName = LiteralQuoteUtils.unquoteColumnName(key);
-      int colIndex = fieldIndex.get(columnName).getSecond();
+      ParquetColumn parquetColumn = fieldIndex.get(columnName);
+      int colIndex = parquetColumn.index;
       RowBufferStats forkedStats = statsMap.get(columnName).forkEmpty();
       forkedStatsMap.put(columnName, forkedStats);
-      ColumnMetadata column = fieldIndex.get(columnName).getFirst();
-      ColumnDescriptor columnDescriptor = schema.getColumns().get(colIndex);
-      PrimitiveType.PrimitiveTypeName typeName =
-          columnDescriptor.getPrimitiveType().getPrimitiveTypeName();
+      ColumnMetadata column = parquetColumn.columnMetadata;
       ParquetValueParser.ParquetBufferValue valueWithSize =
           ParquetValueParser.parseColumnValueToParquet(
-              value, column, typeName, forkedStats, defaultTimezone, curRowIndex);
+              value, column, parquetColumn.type, forkedStats, defaultTimezone, curRowIndex);
       indexedRow[colIndex] = valueWithSize.getValue();
       size += valueWithSize.getSize();
     }
@@ -262,9 +252,9 @@ Object getVectorValueAt(String column, int index) {
     if (data == null) {
       return null;
     }
-    int colIndex = fieldIndex.get(column).getSecond();
+    int colIndex = fieldIndex.get(column).index;
     Object value = data.get(index).get(colIndex);
-    ColumnMetadata columnMetadata = fieldIndex.get(column).getFirst();
+    ColumnMetadata columnMetadata = fieldIndex.get(column).columnMetadata;
     String physicalTypeStr = columnMetadata.getPhysicalType();
     ColumnPhysicalType physicalType = ColumnPhysicalType.valueOf(physicalTypeStr);
     String logicalTypeStr = columnMetadata.getLogicalType();
@@ -315,4 +305,17 @@ void closeInternal() {
   public Flusher<ParquetChunkData> createFlusher() {
     return new ParquetFlusher(schema, enableParquetInternalBuffering);
   }
+
+  private static class ParquetColumn {
+    final ColumnMetadata columnMetadata;
+    final int index;
+    final PrimitiveType.PrimitiveTypeName type;
+
+    private ParquetColumn(
+        ColumnMetadata columnMetadata, int index, PrimitiveType.PrimitiveTypeName type) {
+      this.columnMetadata = columnMetadata;
+      this.index = index;
+      this.type = type;
+    }
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
index 79da87241..c6d0e87c3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetTypeGenerator.java
@@ -42,6 +42,10 @@ public void setParquetType(Type parquetType) {
     public void setMetadata(Map<String, String> metadata) {
       this.metadata = metadata;
     }
+
+    public PrimitiveType.PrimitiveTypeName getPrimitiveTypeName() {
+      return parquetType.asPrimitiveType().getPrimitiveTypeName();
+    }
   }
 
   private static final Set<AbstractRowBuffer.ColumnPhysicalType> TIME_SUPPORTED_PHYSICAL_TYPES =
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 102ee5535..f1a4da9d3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -129,7 +129,6 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
             getFullyQualifiedName(),
             this::collectRowSize,
             channelState,
-            false,
             owningClient != null
                 ? owningClient.getParameterProvider().getEnableParquetInternalBuffering()
                 : ParameterProvider.ENABLE_PARQUET_INTERNAL_BUFFERING_DEFAULT);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 170619d05..b062023fd 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -130,7 +130,6 @@ private AbstractRowBuffer<?> createTestBuffer(OpenChannelRequest.OnErrorOption o
         "test.buffer",
         rs -> {},
         initialState,
-        true,
         enableParquetMemoryOptimization);
   }
 

From 2f7937d89a92c892880016939eadb58842933c84 Mon Sep 17 00:00:00 2001
From: Snyk bot <snyk-bot@snyk.io>
Date: Tue, 18 Apr 2023 06:34:38 +0800
Subject: [PATCH 188/356] [Snyk] Security upgrade net.snowflake:snowflake-jdbc
 from 3.13.25 to 3.13.29 (#476)

fix: pom.xml to reduce vulnerabilities

The following vulnerabilities are fixed with an upgrade:
- https://snyk.io/vuln/SNYK-JAVA-NETSNOWFLAKE-5425048

Co-authored-by: Tyler Jones <59716821+sfc-gh-tjones@users.noreply.github.com>
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index f1fe1e82a..ddd512e0c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -60,7 +60,7 @@
     <shadeBase>net.snowflake.ingest.internal</shadeBase>
     <slf4j.version>1.7.36</slf4j.version>
     <snappy.version>1.1.8.3</snappy.version>
-    <snowjdbc.version>3.13.25</snowjdbc.version>
+    <snowjdbc.version>3.13.29</snowjdbc.version>
     <yetus.version>0.13.0</yetus.version>
   </properties>
 

From 9fdda56ecc780f29596ed5157b09b26f25fa04cc Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Wed, 19 Apr 2023 15:20:39 +0200
Subject: [PATCH 189/356] SNOW-766522 Use caffeine cache (#367)

---
 pom.xml                                       | 20 ++++++++++++++
 .../streaming/internal/LiteralQuoteUtils.java | 27 ++++---------------
 2 files changed, 25 insertions(+), 22 deletions(-)

diff --git a/pom.xml b/pom.xml
index ddd512e0c..d15899315 100644
--- a/pom.xml
+++ b/pom.xml
@@ -266,6 +266,7 @@
 
   <!-- All of our needed dependencies -->
   <dependencies>
+
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-annotations</artifactId>
@@ -280,6 +281,25 @@
       <artifactId>jackson-databind</artifactId>
     </dependency>
 
+    <!-- https://mvnrepository.com/artifact/com.github.ben-manes.caffeine/caffeine -->
+    <!-- We need to use 2.x version, which is compatible with Java 8 -->
+    <dependency>
+      <groupId>com.github.ben-manes.caffeine</groupId>
+      <artifactId>caffeine</artifactId>
+      <version>2.9.3</version>
+      <exclusions>
+        <!-- Dependencies are excluded because they are causing enforcer failures (they are already pulled in by Guava) -->
+        <exclusion>
+          <groupId>com.google.errorprone</groupId>
+          <artifactId>error_prone_annotations</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.checkerframework</groupId>
+          <artifactId>checker-qual</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
     <dependency>
       <groupId>com.google.code.findbugs</groupId>
       <artifactId>jsr305</artifactId>
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/LiteralQuoteUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/LiteralQuoteUtils.java
index 74d97cfa0..12fe5cd3a 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/LiteralQuoteUtils.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/LiteralQuoteUtils.java
@@ -3,12 +3,8 @@
  */
 package net.snowflake.ingest.streaming.internal;
 
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import java.util.concurrent.ExecutionException;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.SFException;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
 
 /**
  * Util class to normalise literals to match server side metadata.
@@ -26,15 +22,9 @@ class LiteralQuoteUtils {
 
   static {
     unquotedColumnNamesCache =
-        CacheBuilder.newBuilder()
+        Caffeine.newBuilder()
             .maximumSize(UNQUOTED_COLUMN_NAME_CACHE_MAX_SIZE)
-            .build(
-                new CacheLoader<String, String>() {
-                  @Override
-                  public String load(String key) {
-                    return unquoteColumnNameInternal(key);
-                  }
-                });
+            .build(LiteralQuoteUtils::unquoteColumnNameInternal);
   }
 
   /**
@@ -42,14 +32,7 @@ public String load(String key) {
    * expensive. If not, it unquotes directly, otherwise it return a value from a loading cache.
    */
   static String unquoteColumnName(String columnName) {
-    try {
-      return unquotedColumnNamesCache.get(columnName);
-    } catch (ExecutionException e) {
-      throw new SFException(
-          e,
-          ErrorCode.INTERNAL_ERROR,
-          String.format("Exception thrown while unquoting column name %s", columnName));
-    }
+    return unquotedColumnNamesCache.get(columnName);
   }
 
   /**

From 31c5136aa0bd4a8cba7441bf5e1db8a282f04a59 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Wed, 19 Apr 2023 16:39:38 +0200
Subject: [PATCH 190/356] SNOW-765525 Add slf4j-simple back to test scope
 (#397)

Co-authored-by: Tyler Jones <59716821+sfc-gh-tjones@users.noreply.github.com>
---
 pom.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/pom.xml b/pom.xml
index d15899315..fcec4c8ec 100644
--- a/pom.xml
+++ b/pom.xml
@@ -431,6 +431,12 @@
       <version>2.0.2</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-simple</artifactId>
+      <version>${slf4j.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

From 3bb5b47eac969d138dc5634536fb133fae6dfdec Mon Sep 17 00:00:00 2001
From: Jay Patel <jay.patel@snowflake.com>
Date: Wed, 19 Apr 2023 14:34:38 -0700
Subject: [PATCH 191/356] NO-SNOW Fix bug for ParquetRowBuffer regarding row
 Index (#477)

* Fix few issues for ParquetRowBufer

* Address Gloria's comments - Added test
---
 .../streaming/internal/ParquetRowBuffer.java  | 14 ++--
 .../internal/ParquetValueParser.java          | 78 +++++++++++--------
 .../streaming/internal/RowBufferTest.java     | 24 +++++-
 3 files changed, 77 insertions(+), 39 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index f2db5dd10..a45a52b93 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -139,7 +139,7 @@ float addRow(
       Map<String, RowBufferStats> statsMap,
       Set<String> formattedInputColumnNames,
       final long insertRowIndex) {
-    return addRow(row, bufferedRowIndex, this::writeRow, statsMap, formattedInputColumnNames);
+    return addRow(row, this::writeRow, statsMap, formattedInputColumnNames, insertRowIndex);
   }
 
   void writeRow(List<Object> row) {
@@ -157,25 +157,27 @@ float addTempRow(
       Map<String, RowBufferStats> statsMap,
       Set<String> formattedInputColumnNames,
       long insertRowIndex) {
-    return addRow(row, curRowIndex, tempData::add, statsMap, formattedInputColumnNames);
+    return addRow(row, tempData::add, statsMap, formattedInputColumnNames, insertRowIndex);
   }
 
   /**
    * Adds a row to the parquet buffer.
    *
    * @param row row to add
-   * @param curRowIndex current row index
    * @param out internal buffer to add to
    * @param statsMap column stats map
    * @param inputColumnNames list of input column names after formatting
+   * @param insertRowsCurrIndex Row index of the input Rows passed in {@link
+   *     net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel#insertRows(Iterable,
+   *     String)}
    * @return row size
    */
   private float addRow(
       Map<String, Object> row,
-      int curRowIndex,
       Consumer<List<Object>> out,
       Map<String, RowBufferStats> statsMap,
-      Set<String> inputColumnNames) {
+      Set<String> inputColumnNames,
+      long insertRowsCurrIndex) {
     Object[] indexedRow = new Object[fieldIndex.size()];
     float size = 0F;
 
@@ -193,7 +195,7 @@ private float addRow(
       ColumnMetadata column = parquetColumn.columnMetadata;
       ParquetValueParser.ParquetBufferValue valueWithSize =
           ParquetValueParser.parseColumnValueToParquet(
-              value, column, parquetColumn.type, forkedStats, defaultTimezone, curRowIndex);
+              value, column, parquetColumn.type, forkedStats, defaultTimezone, insertRowsCurrIndex);
       indexedRow[colIndex] = valueWithSize.getValue();
       size += valueWithSize.getSize();
     }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
index a5785c85d..282a007d4 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetValueParser.java
@@ -75,7 +75,8 @@ float getSize() {
    * @param columnMetadata column metadata
    * @param typeName Parquet primitive type name
    * @param stats column stats to update
-   * @param curRowIndex Row index corresponding the row to parse
+   * @param insertRowsCurrIndex Row index corresponding the row to parse (w.r.t input rows in
+   *     insertRows API, and not buffered row)
    * @return parsed value and byte size of Parquet internal representation
    */
   static ParquetBufferValue parseColumnValueToParquet(
@@ -84,7 +85,7 @@ static ParquetBufferValue parseColumnValueToParquet(
       PrimitiveType.PrimitiveTypeName typeName,
       RowBufferStats stats,
       ZoneId defaultTimezone,
-      int curRowIndex) {
+      long insertRowsCurrIndex) {
     Utils.assertNotNull("Parquet column stats", stats);
     float estimatedParquetSize = 0F;
     estimatedParquetSize += DEFINITION_LEVEL_ENCODING_BYTE_LEN;
@@ -97,7 +98,7 @@ static ParquetBufferValue parseColumnValueToParquet(
         case BOOLEAN:
           int intValue =
               DataValidationUtil.validateAndParseBoolean(
-                  columnMetadata.getName(), value, curRowIndex);
+                  columnMetadata.getName(), value, insertRowsCurrIndex);
           value = intValue > 0;
           stats.addIntValue(BigInteger.valueOf(intValue));
           estimatedParquetSize += BIT_ENCODING_BYTE_LEN;
@@ -111,7 +112,7 @@ static ParquetBufferValue parseColumnValueToParquet(
                   Optional.ofNullable(columnMetadata.getPrecision()).orElse(0),
                   logicalType,
                   physicalType,
-                  curRowIndex);
+                  insertRowsCurrIndex);
           value = intVal;
           stats.addIntValue(BigInteger.valueOf(intVal));
           estimatedParquetSize += 4;
@@ -126,14 +127,15 @@ static ParquetBufferValue parseColumnValueToParquet(
                   logicalType,
                   physicalType,
                   defaultTimezone,
-                  curRowIndex);
+                  insertRowsCurrIndex);
           value = longValue;
           stats.addIntValue(BigInteger.valueOf(longValue));
           estimatedParquetSize += 8;
           break;
         case DOUBLE:
           double doubleValue =
-              DataValidationUtil.validateAndParseReal(columnMetadata.getName(), value, curRowIndex);
+              DataValidationUtil.validateAndParseReal(
+                  columnMetadata.getName(), value, insertRowsCurrIndex);
           value = doubleValue;
           stats.addRealValue(doubleValue);
           estimatedParquetSize += 8;
@@ -141,10 +143,11 @@ static ParquetBufferValue parseColumnValueToParquet(
         case BINARY:
           int length = 0;
           if (logicalType == AbstractRowBuffer.ColumnLogicalType.BINARY) {
-            value = getBinaryValueForLogicalBinary(value, stats, columnMetadata, curRowIndex);
+            value =
+                getBinaryValueForLogicalBinary(value, stats, columnMetadata, insertRowsCurrIndex);
             length = ((byte[]) value).length;
           } else {
-            String str = getBinaryValue(value, stats, columnMetadata, curRowIndex);
+            String str = getBinaryValue(value, stats, columnMetadata, insertRowsCurrIndex);
             value = str;
             if (str != null) {
               length = str.getBytes().length;
@@ -165,7 +168,7 @@ static ParquetBufferValue parseColumnValueToParquet(
                   logicalType,
                   physicalType,
                   defaultTimezone,
-                  curRowIndex);
+                  insertRowsCurrIndex);
           stats.addIntValue(intRep);
           value = getSb16Bytes(intRep);
           estimatedParquetSize += 16;
@@ -196,6 +199,7 @@ static ParquetBufferValue parseColumnValueToParquet(
    * @param precision data type precision
    * @param logicalType Snowflake logical type
    * @param physicalType Snowflake physical type
+   * @param insertRowsCurrIndex Used for logging the row of index given in insertRows API
    * @return parsed int32 value
    */
   private static int getInt32Value(
@@ -205,23 +209,24 @@ private static int getInt32Value(
       Integer precision,
       AbstractRowBuffer.ColumnLogicalType logicalType,
       AbstractRowBuffer.ColumnPhysicalType physicalType,
-      final int curRowIndex) {
+      final long insertRowsCurrIndex) {
     int intVal;
     switch (logicalType) {
       case DATE:
-        intVal = DataValidationUtil.validateAndParseDate(columnName, value, curRowIndex);
+        intVal = DataValidationUtil.validateAndParseDate(columnName, value, insertRowsCurrIndex);
         break;
       case TIME:
         Utils.assertNotNull("Unexpected null scale for TIME data type", scale);
         intVal =
-            DataValidationUtil.validateAndParseTime(columnName, value, scale, curRowIndex)
+            DataValidationUtil.validateAndParseTime(columnName, value, scale, insertRowsCurrIndex)
                 .intValue();
         break;
       case FIXED:
         BigDecimal bigDecimalValue =
-            DataValidationUtil.validateAndParseBigDecimal(columnName, value, curRowIndex);
+            DataValidationUtil.validateAndParseBigDecimal(columnName, value, insertRowsCurrIndex);
         bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
-        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision, curRowIndex);
+        DataValidationUtil.checkValueInRange(
+            bigDecimalValue, scale, precision, insertRowsCurrIndex);
         intVal = bigDecimalValue.intValue();
         break;
       default:
@@ -248,13 +253,13 @@ private static long getInt64Value(
       AbstractRowBuffer.ColumnLogicalType logicalType,
       AbstractRowBuffer.ColumnPhysicalType physicalType,
       ZoneId defaultTimezone,
-      final int curRowIndex) {
+      final long insertRowsCurrIndex) {
     long longValue;
     switch (logicalType) {
       case TIME:
         Utils.assertNotNull("Unexpected null scale for TIME data type", scale);
         longValue =
-            DataValidationUtil.validateAndParseTime(columnName, value, scale, curRowIndex)
+            DataValidationUtil.validateAndParseTime(columnName, value, scale, insertRowsCurrIndex)
                 .longValue();
         break;
       case TIMESTAMP_LTZ:
@@ -262,22 +267,23 @@ private static long getInt64Value(
         boolean trimTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
         longValue =
             DataValidationUtil.validateAndParseTimestamp(
-                    columnName, value, scale, defaultTimezone, trimTimezone, curRowIndex)
+                    columnName, value, scale, defaultTimezone, trimTimezone, insertRowsCurrIndex)
                 .toBinary(false)
                 .longValue();
         break;
       case TIMESTAMP_TZ:
         longValue =
             DataValidationUtil.validateAndParseTimestamp(
-                    columnName, value, scale, defaultTimezone, false, curRowIndex)
+                    columnName, value, scale, defaultTimezone, false, insertRowsCurrIndex)
                 .toBinary(true)
                 .longValue();
         break;
       case FIXED:
         BigDecimal bigDecimalValue =
-            DataValidationUtil.validateAndParseBigDecimal(columnName, value, curRowIndex);
+            DataValidationUtil.validateAndParseBigDecimal(columnName, value, insertRowsCurrIndex);
         bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
-        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision, curRowIndex);
+        DataValidationUtil.checkValueInRange(
+            bigDecimalValue, scale, precision, insertRowsCurrIndex);
         longValue = bigDecimalValue.longValue();
         break;
       default:
@@ -304,24 +310,25 @@ private static BigInteger getSb16Value(
       AbstractRowBuffer.ColumnLogicalType logicalType,
       AbstractRowBuffer.ColumnPhysicalType physicalType,
       ZoneId defaultTimezone,
-      final int curRowIndex) {
+      final long insertRowsCurrIndex) {
     switch (logicalType) {
       case TIMESTAMP_TZ:
         return DataValidationUtil.validateAndParseTimestamp(
-                columnName, value, scale, defaultTimezone, false, curRowIndex)
+                columnName, value, scale, defaultTimezone, false, insertRowsCurrIndex)
             .toBinary(true);
       case TIMESTAMP_LTZ:
       case TIMESTAMP_NTZ:
         boolean trimTimezone = logicalType == AbstractRowBuffer.ColumnLogicalType.TIMESTAMP_NTZ;
         return DataValidationUtil.validateAndParseTimestamp(
-                columnName, value, scale, defaultTimezone, trimTimezone, curRowIndex)
+                columnName, value, scale, defaultTimezone, trimTimezone, insertRowsCurrIndex)
             .toBinary(false);
       case FIXED:
         BigDecimal bigDecimalValue =
-            DataValidationUtil.validateAndParseBigDecimal(columnName, value, curRowIndex);
+            DataValidationUtil.validateAndParseBigDecimal(columnName, value, insertRowsCurrIndex);
         // explicitly match the BigDecimal input scale with the Snowflake data type scale
         bigDecimalValue = bigDecimalValue.setScale(scale, RoundingMode.HALF_UP);
-        DataValidationUtil.checkValueInRange(bigDecimalValue, scale, precision, curRowIndex);
+        DataValidationUtil.checkValueInRange(
+            bigDecimalValue, scale, precision, insertRowsCurrIndex);
         return bigDecimalValue.unscaledValue();
       default:
         throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
@@ -351,10 +358,14 @@ static byte[] getSb16Bytes(BigInteger intRep) {
    * @param value value to parse
    * @param stats column stats to update
    * @param columnMetadata column metadata
+   * @param insertRowsCurrIndex Used for logging the row of index given in insertRows API
    * @return string representation
    */
   private static String getBinaryValue(
-      Object value, RowBufferStats stats, ColumnMetadata columnMetadata, final int curRowIndex) {
+      Object value,
+      RowBufferStats stats,
+      ColumnMetadata columnMetadata,
+      final long insertRowsCurrIndex) {
     AbstractRowBuffer.ColumnLogicalType logicalType =
         AbstractRowBuffer.ColumnLogicalType.valueOf(columnMetadata.getLogicalType());
     String str;
@@ -363,17 +374,17 @@ private static String getBinaryValue(
         case OBJECT:
           str =
               DataValidationUtil.validateAndParseObject(
-                  columnMetadata.getName(), value, curRowIndex);
+                  columnMetadata.getName(), value, insertRowsCurrIndex);
           break;
         case VARIANT:
           str =
               DataValidationUtil.validateAndParseVariant(
-                  columnMetadata.getName(), value, curRowIndex);
+                  columnMetadata.getName(), value, insertRowsCurrIndex);
           break;
         case ARRAY:
           str =
               DataValidationUtil.validateAndParseArray(
-                  columnMetadata.getName(), value, curRowIndex);
+                  columnMetadata.getName(), value, insertRowsCurrIndex);
           break;
         default:
           throw new SFException(
@@ -386,7 +397,7 @@ private static String getBinaryValue(
               columnMetadata.getName(),
               value,
               Optional.of(maxLengthString).map(Integer::parseInt),
-              curRowIndex);
+              insertRowsCurrIndex);
       stats.addStrValue(str);
     }
     return str;
@@ -400,14 +411,17 @@ private static String getBinaryValue(
    * @return byte array representation
    */
   private static byte[] getBinaryValueForLogicalBinary(
-      Object value, RowBufferStats stats, ColumnMetadata columnMetadata, final int curRowIndex) {
+      Object value,
+      RowBufferStats stats,
+      ColumnMetadata columnMetadata,
+      final long insertRowsCurrIndex) {
     String maxLengthString = columnMetadata.getByteLength().toString();
     byte[] bytes =
         DataValidationUtil.validateAndParseBinary(
             columnMetadata.getName(),
             value,
             Optional.of(maxLengthString).map(Integer::parseInt),
-            curRowIndex);
+            insertRowsCurrIndex);
     stats.addBinaryValue(bytes);
     return bytes;
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index b062023fd..55df1eb21 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -13,6 +13,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.utils.Constants;
@@ -281,7 +282,7 @@ public void testStringLength() {
   }
 
   @Test
-  public void testRowIndexWithMultipleRowsWithErrorr() {
+  public void testRowIndexWithMultipleRowsWithError() {
     List<Map<String, Object>> rows = new ArrayList<>();
     Map<String, Object> row = new HashMap<>();
 
@@ -303,6 +304,27 @@ public void testRowIndexWithMultipleRowsWithErrorr() {
     // second row out of the rows we sent was having bad data.
     // so InsertError corresponds to second row.
     Assert.assertEquals(1, response.getInsertErrors().get(0).getRowIndex());
+
+    Assert.assertTrue(response.getInsertErrors().get(0).getException() != null);
+
+    Assert.assertTrue(
+        Objects.equals(
+            response.getInsertErrors().get(0).getException().getVendorCode(),
+            ErrorCode.INVALID_VALUE_ROW.getMessageCode()));
+
+    Assert.assertEquals(1, response.getInsertErrors().get(0).getRowIndex());
+
+    Assert.assertTrue(
+        response
+            .getInsertErrors()
+            .get(0)
+            .getException()
+            .getMessage()
+            .equalsIgnoreCase(
+                "The given row cannot be converted to the internal format due to invalid value:"
+                    + " Value cannot be ingested into Snowflake column COLCHAR of type STRING, Row"
+                    + " Index: 1, reason: String too long: length=22 characters maxLength=11"
+                    + " characters"));
   }
 
   private void testStringLengthHelper(AbstractRowBuffer<?> rowBuffer) {

From 295e8be6dba0ec4d65b693f076e285aec66de9e5 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Wed, 19 Apr 2023 17:31:50 -0700
Subject: [PATCH 192/356] personal nits, it failing bc need server change first

---
 .../streaming/internal/BlobLatencies.java     |  9 +--
 .../streaming/internal/FlushService.java      | 53 ++++++++---------
 .../streaming/internal/StreamingIngestIT.java | 57 ++++++++++---------
 3 files changed, 63 insertions(+), 56 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
index acef2543f..05288ead9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
@@ -15,6 +15,7 @@ class BlobLatencies {
   private Long buildDurationMs;
   private Long uploadDurationMs;
 
+  // flush and register duration cannot be calculated in the client sdk we pass the start time because the end time is when the request hits the server
   private Long flushStartMs;
   private Long registerStartMs;
 
@@ -26,22 +27,22 @@ public BlobLatencies() {
     this.registerStartMs = DEFAULT_BLOB_LATENCY;
   }
 
-  @JsonProperty("build_latency_ms")
+  @JsonProperty("build_duration_ms")
   long getBuildDurationMs() {
     return this.buildDurationMs;
   }
 
-  @JsonProperty("upload_latency_ms")
+  @JsonProperty("upload_duration_ms")
   long getUploadDurationMs() {
     return this.uploadDurationMs;
   }
 
-  @JsonProperty("flush_start_timestamp")
+  @JsonProperty("flush_start_ms")
   long getFlushStartMs() {
     return this.flushStartMs;
   }
 
-  @JsonProperty("register_start_timestamp")
+  @JsonProperty("register_start_ms")
   long getRegisterStartMs() {
     return this.registerStartMs;
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 7ab1fd1c5..edcc23f45 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -4,14 +4,21 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
-import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
-import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
-
 import com.codahale.metrics.Timer;
+import net.snowflake.client.jdbc.SnowflakeSQLException;
+import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.util.VisibleForTesting;
+import org.apache.arrow.vector.VectorSchemaRoot;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.security.InvalidAlgorithmParameterException;
@@ -36,19 +43,13 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import net.snowflake.client.jdbc.SnowflakeSQLException;
-import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.util.VisibleForTesting;
-import org.apache.arrow.vector.VectorSchemaRoot;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
+import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
+import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
 /**
  * Responsible for flushing data from client to Snowflake tables. When a flush is triggered, it will
@@ -518,12 +519,6 @@ BlobMetadata upload(
     Timer.Context uploadContext = Utils.createTimerContext(this.owningClient.uploadLatency);
     this.targetStage.put(filePath, blob);
 
-    logger.logInfo(
-        "Finish uploading file={}, size={}, timeInMillis={}",
-        filePath,
-        blob.length,
-        System.currentTimeMillis() - startTime);
-
     if (uploadContext != null) {
       blobLatencies.setUploadDurationMs(uploadContext);
       this.owningClient.uploadThroughput.mark(blob.length);
@@ -532,6 +527,12 @@ BlobMetadata upload(
           metadata.stream().mapToLong(i -> i.getEpInfo().getRowCount()).sum());
     }
 
+    logger.logInfo(
+        "Finish uploading file={}, size={}, timeInMillis={}",
+        filePath,
+        blob.length,
+        System.currentTimeMillis() - startTime);
+
     return BlobMetadata.createBlobMetadata(
         filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata, blobLatencies);
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 3ff33e43b..e1f53736a 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -1,8 +1,26 @@
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.*;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.core.Is.is;
+import net.snowflake.ingest.TestUtils;
+import net.snowflake.ingest.connection.RequestBuilder;
+import net.snowflake.ingest.streaming.InsertValidationResponse;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.HttpUtil;
+import net.snowflake.ingest.utils.ParameterProvider;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.SnowflakeURL;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -28,27 +46,13 @@
 import java.util.function.Consumer;
 import java.util.function.IntConsumer;
 import java.util.stream.IntStream;
-import net.snowflake.ingest.TestUtils;
-import net.snowflake.ingest.connection.RequestBuilder;
-import net.snowflake.ingest.streaming.InsertValidationResponse;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.HttpUtil;
-import net.snowflake.ingest.utils.ParameterProvider;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.SnowflakeURL;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.mockito.ArgumentMatchers;
-import org.mockito.Mockito;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
+import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
+import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.ROLE;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
 
 /** Example streaming ingest sdk integration test */
 @RunWith(Parameterized.class)
@@ -121,13 +125,14 @@ public void afterAll() throws Exception {
   @Test
   public void testSimpleIngest() throws Exception {
     // TODO @rcheng - dont want to change factory, so inject
+    SnowflakeURL url = new SnowflakeURL(TestUtils.getAccountURL());
     RequestBuilder requestBuilder =
         Mockito.spy(
             new RequestBuilder(
-                new SnowflakeURL(TestUtils.getAccountURL()),
+                url,
                 TestUtils.getUser(),
                 TestUtils.getKeyPair(),
-                HttpUtil.getHttpClient(),
+                HttpUtil.getHttpClient(url.getAccount()),
                 "testrequestbuilder"));
     client.injectRequestBuilder(requestBuilder);
 

From d98138d2dc1db2edafe5ecaec460ce56ff8f33dc Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Wed, 19 Apr 2023 17:31:52 -0700
Subject: [PATCH 193/356] autoformatting

---
 .../streaming/internal/BlobLatencies.java     |  3 +-
 .../streaming/internal/FlushService.java      | 41 +++++++-------
 .../streaming/internal/StreamingIngestIT.java | 55 +++++++++----------
 3 files changed, 49 insertions(+), 50 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
index 05288ead9..b305241cc 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
@@ -15,7 +15,8 @@ class BlobLatencies {
   private Long buildDurationMs;
   private Long uploadDurationMs;
 
-  // flush and register duration cannot be calculated in the client sdk we pass the start time because the end time is when the request hits the server
+  // flush and register duration cannot be calculated in the client sdk we pass the start time
+  // because the end time is when the request hits the server
   private Long flushStartMs;
   private Long registerStartMs;
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index edcc23f45..46cc9e511 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -4,21 +4,14 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import com.codahale.metrics.Timer;
-import net.snowflake.client.jdbc.SnowflakeSQLException;
-import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.util.VisibleForTesting;
-import org.apache.arrow.vector.VectorSchemaRoot;
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
+import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
+import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
+import com.codahale.metrics.Timer;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.security.InvalidAlgorithmParameterException;
@@ -43,13 +36,19 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
-import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
-import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import net.snowflake.client.jdbc.SnowflakeSQLException;
+import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.util.VisibleForTesting;
+import org.apache.arrow.vector.VectorSchemaRoot;
 
 /**
  * Responsible for flushing data from client to Snowflake tables. When a flush is triggered, it will
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index e1f53736a..ed1532848 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -1,26 +1,11 @@
 package net.snowflake.ingest.streaming.internal;
 
-import net.snowflake.ingest.TestUtils;
-import net.snowflake.ingest.connection.RequestBuilder;
-import net.snowflake.ingest.streaming.InsertValidationResponse;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.HttpUtil;
-import net.snowflake.ingest.utils.ParameterProvider;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.SnowflakeURL;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.mockito.ArgumentMatchers;
-import org.mockito.Mockito;
+import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
+import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
+import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.ROLE;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -46,13 +31,27 @@
 import java.util.function.Consumer;
 import java.util.function.IntConsumer;
 import java.util.stream.IntStream;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
-import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
-import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.ROLE;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.core.Is.is;
+import net.snowflake.ingest.TestUtils;
+import net.snowflake.ingest.connection.RequestBuilder;
+import net.snowflake.ingest.streaming.InsertValidationResponse;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.HttpUtil;
+import net.snowflake.ingest.utils.ParameterProvider;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.SnowflakeURL;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
 
 /** Example streaming ingest sdk integration test */
 @RunWith(Parameterized.class)

From e0a840f6c72aef659b9ef549bdf925473d7c3067 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Fri, 21 Apr 2023 13:57:54 -0700
Subject: [PATCH 194/356] rename and flush timer move

---
 .../streaming/internal/BlobBuilder.java       | 45 ++++++++-------
 .../streaming/internal/BlobMetadata.java      | 23 ++++----
 .../{BlobLatencies.java => BlobStats.java}    | 20 ++-----
 .../streaming/internal/FlushService.java      | 57 ++++++++++---------
 .../streaming/internal/RegisterService.java   | 17 +++---
 ...nowflakeStreamingIngestClientInternal.java |  2 +-
 .../streaming/internal/FlushServiceTest.java  |  4 +-
 .../streaming/internal/StreamingIngestIT.java | 57 ++++++++++---------
 8 files changed, 111 insertions(+), 114 deletions(-)
 rename src/main/java/net/snowflake/ingest/streaming/internal/{BlobLatencies.java => BlobStats.java} (77%)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
index e4f0b807e..3a0c9215f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
@@ -4,18 +4,17 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
-import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
-import static net.snowflake.ingest.utils.Utils.toByteArray;
-
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.Cryptor;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import org.apache.commons.codec.binary.Hex;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.security.InvalidAlgorithmParameterException;
@@ -26,14 +25,16 @@
 import java.util.List;
 import java.util.zip.CRC32;
 import java.util.zip.GZIPOutputStream;
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.Cryptor;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import org.apache.commons.codec.binary.Hex;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
+import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
+import static net.snowflake.ingest.utils.Utils.toByteArray;
 
 /**
  * Build a single blob file that contains file header plus data. The header will be a
@@ -157,7 +158,7 @@ static <T> Blob constructBlobAndMetadata(
     // Build blob file bytes
     byte[] blobBytes =
         buildBlob(chunksMetadataList, chunksDataList, crc.getValue(), curDataSize, bdecVersion);
-    return new Blob(blobBytes, chunksMetadataList, new BlobLatencies());
+    return new Blob(blobBytes, chunksMetadataList, new BlobStats());
   }
 
   /**
@@ -327,12 +328,12 @@ static String computeMD5(byte[] data, int length) throws NoSuchAlgorithmExceptio
   static class Blob {
     final byte[] blobBytes;
     final List<ChunkMetadata> chunksMetadataList;
-    final BlobLatencies blobLatencies;
+    final BlobStats blobStats;
 
-    Blob(byte[] blobBytes, List<ChunkMetadata> chunksMetadataList, BlobLatencies blobLatencies) {
+    Blob(byte[] blobBytes, List<ChunkMetadata> chunksMetadataList, BlobStats blobStats) {
       this.blobBytes = blobBytes;
       this.chunksMetadataList = chunksMetadataList;
-      this.blobLatencies = blobLatencies;
+      this.blobStats = blobStats;
     }
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index 7b4973b62..d86364601 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -6,23 +6,24 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import java.util.List;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ParameterProvider;
 import org.apache.arrow.util.VisibleForTesting;
 
+import java.util.List;
+
 /** Metadata for a blob that sends to Snowflake as part of the register blob request */
 class BlobMetadata {
   private final String path;
   private final String md5;
   private final Constants.BdecVersion bdecVersion;
   private final List<ChunkMetadata> chunks;
-  private final BlobLatencies blobLatencies;
+  private final BlobStats blobStats;
 
   // used for testing only
   @VisibleForTesting
-  BlobMetadata(String path, String md5, List<ChunkMetadata> chunks, BlobLatencies blobLatencies) {
-    this(path, md5, ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT, chunks, blobLatencies);
+  BlobMetadata(String path, String md5, List<ChunkMetadata> chunks, BlobStats blobStats) {
+    this(path, md5, ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT, chunks, blobStats);
   }
 
   BlobMetadata(
@@ -30,12 +31,12 @@ class BlobMetadata {
       String md5,
       Constants.BdecVersion bdecVersion,
       List<ChunkMetadata> chunks,
-      BlobLatencies blobLatencies) {
+      BlobStats blobStats) {
     this.path = path;
     this.md5 = md5;
     this.bdecVersion = bdecVersion;
     this.chunks = chunks;
-    this.blobLatencies = blobLatencies;
+    this.blobStats = blobStats;
   }
 
   @JsonIgnore
@@ -63,9 +64,9 @@ byte getVersionByte() {
     return bdecVersion.toByte();
   }
 
-  @JsonProperty("blob_latencies")
-  BlobLatencies getBlobLatencies() {
-    return this.blobLatencies;
+  @JsonProperty("blob_stats")
+  BlobStats getBlobStats() {
+    return this.blobStats;
   }
 
   /** Create {@link BlobMetadata}. */
@@ -74,7 +75,7 @@ static BlobMetadata createBlobMetadata(
       String md5,
       Constants.BdecVersion bdecVersion,
       List<ChunkMetadata> chunks,
-      BlobLatencies blobLatencies) {
-    return new BlobMetadata(path, md5, bdecVersion, chunks, blobLatencies);
+      BlobStats blobStats) {
+    return new BlobMetadata(path, md5, bdecVersion, chunks, blobStats);
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobStats.java
similarity index 77%
rename from src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
rename to src/main/java/net/snowflake/ingest/streaming/internal/BlobStats.java
index b305241cc..ccd2f9eeb 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobLatencies.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobStats.java
@@ -9,24 +9,14 @@
 import java.util.concurrent.TimeUnit;
 
 /** Latency information for a blob */
-class BlobLatencies {
-  public static final Long DEFAULT_BLOB_LATENCY = null;
-
-  private Long buildDurationMs;
-  private Long uploadDurationMs;
+class BlobStats {
+  private long buildDurationMs;
+  private long uploadDurationMs;
 
   // flush and register duration cannot be calculated in the client sdk we pass the start time
   // because the end time is when the request hits the server
-  private Long flushStartMs;
-  private Long registerStartMs;
-
-  public BlobLatencies() {
-    this.buildDurationMs = DEFAULT_BLOB_LATENCY;
-    this.uploadDurationMs = DEFAULT_BLOB_LATENCY;
-
-    this.flushStartMs = DEFAULT_BLOB_LATENCY;
-    this.registerStartMs = DEFAULT_BLOB_LATENCY;
-  }
+  private long flushStartMs;
+  private long registerStartMs;
 
   @JsonProperty("build_duration_ms")
   long getBuildDurationMs() {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 46cc9e511..a4da7cce1 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -4,14 +4,21 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
-import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
-import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
-
 import com.codahale.metrics.Timer;
+import net.snowflake.client.jdbc.SnowflakeSQLException;
+import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.util.VisibleForTesting;
+import org.apache.arrow.vector.VectorSchemaRoot;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.security.InvalidAlgorithmParameterException;
@@ -36,19 +43,13 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import net.snowflake.client.jdbc.SnowflakeSQLException;
-import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.util.VisibleForTesting;
-import org.apache.arrow.vector.VectorSchemaRoot;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
+import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
+import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
 /**
  * Responsible for flushing data from client to Snowflake tables. When a flush is triggered, it will
@@ -421,6 +422,7 @@ void distributeFlushTasks() {
         // client. See method getFilePath() below.
         this.counter.decrementAndGet();
       } else {
+        long flushStartMs = System.currentTimeMillis();
         if (this.owningClient.flushLatency != null) {
           latencyTimerContextMap.putIfAbsent(filePath, this.owningClient.flushLatency.time());
         }
@@ -430,7 +432,9 @@ void distributeFlushTasks() {
                 CompletableFuture.supplyAsync(
                     () -> {
                       try {
-                        return buildAndUpload(filePath, blobData);
+                        BlobMetadata blobMetadata = buildAndUpload(filePath, blobData);
+                        blobMetadata.getBlobStats().setFlushStartMs(flushStartMs);
+                        return blobMetadata;
                       } catch (Throwable e) {
                         Throwable ex = e.getCause() == null ? e : e.getCause();
                         String errorMessage =
@@ -496,9 +500,9 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
     // Construct the blob along with the metadata of the blob
     BlobBuilder.Blob blob = BlobBuilder.constructBlobAndMetadata(filePath, blobData, bdecVersion);
 
-    blob.blobLatencies.setBuildDurationMs(buildContext);
+    blob.blobStats.setBuildDurationMs(buildContext);
 
-    return upload(filePath, blob.blobBytes, blob.chunksMetadataList, blob.blobLatencies);
+    return upload(filePath, blob.blobBytes, blob.chunksMetadataList, blob.blobStats);
   }
 
   /**
@@ -507,10 +511,11 @@ BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData
    * @param filePath full path of the blob file
    * @param blob blob data
    * @param metadata a list of chunk metadata
+   * @param blobStats an object to track latencies and other stats of the blob
    * @return BlobMetadata object used to create the register blob request
    */
   BlobMetadata upload(
-      String filePath, byte[] blob, List<ChunkMetadata> metadata, BlobLatencies blobLatencies)
+      String filePath, byte[] blob, List<ChunkMetadata> metadata, BlobStats blobStats)
       throws NoSuchAlgorithmException {
     logger.logInfo("Start uploading file={}, size={}", filePath, blob.length);
     long startTime = System.currentTimeMillis();
@@ -519,7 +524,7 @@ BlobMetadata upload(
     this.targetStage.put(filePath, blob);
 
     if (uploadContext != null) {
-      blobLatencies.setUploadDurationMs(uploadContext);
+      blobStats.setUploadDurationMs(uploadContext);
       this.owningClient.uploadThroughput.mark(blob.length);
       this.owningClient.blobSizeHistogram.update(blob.length);
       this.owningClient.blobRowCountHistogram.update(
@@ -533,7 +538,7 @@ BlobMetadata upload(
         System.currentTimeMillis() - startTime);
 
     return BlobMetadata.createBlobMetadata(
-        filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata, blobLatencies);
+        filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata, blobStats);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index d00d4705c..f8acf0e6a 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -4,10 +4,11 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
-
 import com.codahale.metrics.Timer;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.Utils;
+
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -17,9 +18,9 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.Utils;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
 /**
  * Register one or more blobs to the targeted Snowflake table, it will be done using the dedicated
@@ -197,9 +198,7 @@ List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyT
                 Utils.createTimerContext(this.owningClient.registerLatency);
 
             for (BlobMetadata blobMetadata : blobs) {
-              long currentTimeMs = System.currentTimeMillis();
-              blobMetadata.getBlobLatencies().setRegisterStartMs(currentTimeMs);
-              blobMetadata.getBlobLatencies().setFlushStartMs(currentTimeMs);
+              blobMetadata.getBlobStats().setRegisterStartMs(System.currentTimeMillis());
             }
 
             // Register the blobs, and invalidate any channels that return a failure status code
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 523c71841..4a87e2bd2 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -584,7 +584,7 @@ List<BlobMetadata> getRetryBlobs(
                     blobMetadata.getMD5(),
                     blobMetadata.getVersion(),
                     relevantChunks,
-                    blobMetadata.getBlobLatencies()));
+                    blobMetadata.getBlobStats()));
           }
         });
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 94d7cb320..b4306b888 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -630,9 +630,9 @@ public void testBuildAndUpload() throws Exception {
 
     Assert.assertEquals(BlobBuilder.computeMD5(blobCaptor.getValue()), blobMetadata.getMD5());
     Assert.assertEquals(
-        expectedBuildLatencyMs, blobMetadata.getBlobLatencies().getBuildDurationMs());
+        expectedBuildLatencyMs, blobMetadata.getBlobStats().getBuildDurationMs());
     Assert.assertEquals(
-        expectedUploadLatencyMs, blobMetadata.getBlobLatencies().getUploadDurationMs());
+        expectedUploadLatencyMs, blobMetadata.getBlobStats().getUploadDurationMs());
 
     Assert.assertEquals(
         expectedChunkEpInfo.getRowCount(), metadataResult.getEpInfo().getRowCount());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index ed1532848..960e60293 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -1,11 +1,26 @@
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
-import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
-import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.ROLE;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.core.Is.is;
+import net.snowflake.ingest.TestUtils;
+import net.snowflake.ingest.connection.RequestBuilder;
+import net.snowflake.ingest.streaming.InsertValidationResponse;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.HttpUtil;
+import net.snowflake.ingest.utils.ParameterProvider;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.SnowflakeURL;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -31,27 +46,13 @@
 import java.util.function.Consumer;
 import java.util.function.IntConsumer;
 import java.util.stream.IntStream;
-import net.snowflake.ingest.TestUtils;
-import net.snowflake.ingest.connection.RequestBuilder;
-import net.snowflake.ingest.streaming.InsertValidationResponse;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.HttpUtil;
-import net.snowflake.ingest.utils.ParameterProvider;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.SnowflakeURL;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.mockito.ArgumentMatchers;
-import org.mockito.Mockito;
+
+import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
+import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
+import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.ROLE;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
 
 /** Example streaming ingest sdk integration test */
 @RunWith(Parameterized.class)
@@ -156,7 +157,7 @@ public void testSimpleIngest() throws Exception {
 
     // verify expected request sent to server
     String[] expectedPayloadParams = {
-      "request_id", "blobs", "role", "flush_start_timestamp", "register_start_timestamp"
+      "request_id", "blobs", "role", "blob_stats"
     };
     for (String expectedParam : expectedPayloadParams) {
       Mockito.verify(requestBuilder)

From 19ef10d2b27e3126e1b5f9695fe4c92cf9a96bad Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Fri, 21 Apr 2023 13:57:56 -0700
Subject: [PATCH 195/356] autoformatting

---
 .../streaming/internal/BlobBuilder.java       | 37 ++++++------
 .../streaming/internal/BlobMetadata.java      |  3 +-
 .../streaming/internal/FlushService.java      | 41 +++++++------
 .../streaming/internal/RegisterService.java   | 13 ++--
 .../streaming/internal/FlushServiceTest.java  |  6 +-
 .../streaming/internal/StreamingIngestIT.java | 59 +++++++++----------
 6 files changed, 75 insertions(+), 84 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
index 3a0c9215f..e1ce6b8bd 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
@@ -4,17 +4,18 @@
 
 package net.snowflake.ingest.streaming.internal;
 
+import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
+import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
+import static net.snowflake.ingest.utils.Utils.toByteArray;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.Cryptor;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import org.apache.commons.codec.binary.Hex;
-
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.security.InvalidAlgorithmParameterException;
@@ -25,16 +26,14 @@
 import java.util.List;
 import java.util.zip.CRC32;
 import java.util.zip.GZIPOutputStream;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_CHECKSUM_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_CHUNK_METADATA_LENGTH_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.BLOB_FILE_SIZE_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
-import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
-import static net.snowflake.ingest.utils.Utils.toByteArray;
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.Cryptor;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import org.apache.commons.codec.binary.Hex;
 
 /**
  * Build a single blob file that contains file header plus data. The header will be a
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index d86364601..392bd53fd 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -6,12 +6,11 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ParameterProvider;
 import org.apache.arrow.util.VisibleForTesting;
 
-import java.util.List;
-
 /** Metadata for a blob that sends to Snowflake as part of the register blob request */
 class BlobMetadata {
   private final String path;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index a4da7cce1..0765ed230 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -4,21 +4,14 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import com.codahale.metrics.Timer;
-import net.snowflake.client.jdbc.SnowflakeSQLException;
-import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.util.VisibleForTesting;
-import org.apache.arrow.vector.VectorSchemaRoot;
+import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
+import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
+import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
+import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
+import com.codahale.metrics.Timer;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.security.InvalidAlgorithmParameterException;
@@ -43,13 +36,19 @@
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_EXTENSION_TYPE;
-import static net.snowflake.ingest.utils.Constants.DISABLE_BACKGROUND_FLUSH;
-import static net.snowflake.ingest.utils.Constants.MAX_BLOB_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.MAX_THREAD_COUNT;
-import static net.snowflake.ingest.utils.Constants.THREAD_SHUTDOWN_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import net.snowflake.client.jdbc.SnowflakeSQLException;
+import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.Utils;
+import org.apache.arrow.util.VisibleForTesting;
+import org.apache.arrow.vector.VectorSchemaRoot;
 
 /**
  * Responsible for flushing data from client to Snowflake tables. When a flush is triggered, it will
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index f8acf0e6a..a48dbc469 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -4,11 +4,10 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import com.codahale.metrics.Timer;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.Utils;
+import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
+import com.codahale.metrics.Timer;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -18,9 +17,9 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_UPLOAD_TIMEOUT_IN_SEC;
-import static net.snowflake.ingest.utils.Utils.getStackTrace;
+import net.snowflake.ingest.utils.Logging;
+import net.snowflake.ingest.utils.Pair;
+import net.snowflake.ingest.utils.Utils;
 
 /**
  * Register one or more blobs to the targeted Snowflake table, it will be done using the dedicated
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index b4306b888..338d8665a 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -629,10 +629,8 @@ public void testBuildAndUpload() throws Exception {
     List<ChannelMetadata> channelMetadataResult = metadataResult.getChannels();
 
     Assert.assertEquals(BlobBuilder.computeMD5(blobCaptor.getValue()), blobMetadata.getMD5());
-    Assert.assertEquals(
-        expectedBuildLatencyMs, blobMetadata.getBlobStats().getBuildDurationMs());
-    Assert.assertEquals(
-        expectedUploadLatencyMs, blobMetadata.getBlobStats().getUploadDurationMs());
+    Assert.assertEquals(expectedBuildLatencyMs, blobMetadata.getBlobStats().getBuildDurationMs());
+    Assert.assertEquals(expectedUploadLatencyMs, blobMetadata.getBlobStats().getUploadDurationMs());
 
     Assert.assertEquals(
         expectedChunkEpInfo.getRowCount(), metadataResult.getEpInfo().getRowCount());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 960e60293..4be4b254e 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -1,26 +1,11 @@
 package net.snowflake.ingest.streaming.internal;
 
-import net.snowflake.ingest.TestUtils;
-import net.snowflake.ingest.connection.RequestBuilder;
-import net.snowflake.ingest.streaming.InsertValidationResponse;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
-import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
-import net.snowflake.ingest.utils.Constants;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.HttpUtil;
-import net.snowflake.ingest.utils.ParameterProvider;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.SnowflakeURL;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.mockito.ArgumentMatchers;
-import org.mockito.Mockito;
+import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
+import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
+import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.ROLE;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -46,13 +31,27 @@
 import java.util.function.Consumer;
 import java.util.function.IntConsumer;
 import java.util.stream.IntStream;
-
-import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
-import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
-import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.ROLE;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.core.Is.is;
+import net.snowflake.ingest.TestUtils;
+import net.snowflake.ingest.connection.RequestBuilder;
+import net.snowflake.ingest.streaming.InsertValidationResponse;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.HttpUtil;
+import net.snowflake.ingest.utils.ParameterProvider;
+import net.snowflake.ingest.utils.SFException;
+import net.snowflake.ingest.utils.SnowflakeURL;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
 
 /** Example streaming ingest sdk integration test */
 @RunWith(Parameterized.class)
@@ -156,9 +155,7 @@ public void testSimpleIngest() throws Exception {
     channel1.close().get();
 
     // verify expected request sent to server
-    String[] expectedPayloadParams = {
-      "request_id", "blobs", "role", "blob_stats"
-    };
+    String[] expectedPayloadParams = {"request_id", "blobs", "role", "blob_stats"};
     for (String expectedParam : expectedPayloadParams) {
       Mockito.verify(requestBuilder)
           .generateStreamingIngestPostRequest(

From 8ba510de1574094c0dbf4b911f8664db66bf9c52 Mon Sep 17 00:00:00 2001
From: Tyler Jones <59716821+sfc-gh-tjones@users.noreply.github.com>
Date: Wed, 19 Apr 2023 16:57:15 -0700
Subject: [PATCH 196/356] Adds back provided scope for slf4j-api (#431)

Co-authored-by: Toby Zhang <toby.zhang@snowflake.com>
---
 pom.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pom.xml b/pom.xml
index fcec4c8ec..a5ed47d3a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -384,6 +384,7 @@
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
+      <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.arrow</groupId>

From 7589d725377ca6248e904b99b0b72289f4da56ec Mon Sep 17 00:00:00 2001
From: Jay Patel <jay.patel@snowflake.com>
Date: Thu, 20 Apr 2023 08:28:17 -0700
Subject: [PATCH 197/356] SNOW-745975 Run tests against Azure and GCP (#478)

* SNOW-745975 Run tests against Azure and GCP

* Add matrix instead of one more workflow

* Rename Workflow top level name, add better name for build

- Also modify the required merge gate name since old job name doesnt run
  anymore
---
 .github/workflows/End2EndTest.yml |  12 ++++++++----
 profile_azure.json.gpg            | Bin 0 -> 1593 bytes
 profile_gcp.json.gpg              | Bin 0 -> 1594 bytes
 scripts/decrypt_secret.sh         |  20 ++++++++++++++++++--
 4 files changed, 26 insertions(+), 6 deletions(-)
 create mode 100644 profile_azure.json.gpg
 create mode 100644 profile_gcp.json.gpg

diff --git a/.github/workflows/End2EndTest.yml b/.github/workflows/End2EndTest.yml
index 5cd5d3fb9..be7f13442 100644
--- a/.github/workflows/End2EndTest.yml
+++ b/.github/workflows/End2EndTest.yml
@@ -1,4 +1,4 @@
-name: Snowpipe Java SDK Tests
+name: Snowpipe Java SDK
 
 on:
   push:
@@ -8,10 +8,13 @@ on:
 
 jobs:
   build:
+    name: Build & Test - JDK ${{ matrix.java }}, Cloud ${{ matrix.snowflake_cloud }}
     runs-on: ubuntu-20.04
     strategy:
+      fail-fast: false # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast
       matrix:
         java: [ 8 ]
+        snowflake_cloud: [ 'AWS', 'AZURE', 'GCP' ]
     steps:
       - name: Checkout Code
         uses: actions/checkout@v2
@@ -20,13 +23,13 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
 
-      - name: Decrypt profile.json
+      - name: Decrypt profile.json for Cloud ${{ matrix.snowflake_cloud }}
         env:
           DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }}
         run: |
-          ./scripts/decrypt_secret.sh
+          ./scripts/decrypt_secret.sh ${{ matrix.snowflake_cloud }}
 
-      - name: Unit & Integration Test
+      - name: Unit & Integration Test against Cloud ${{ matrix.snowflake_cloud }}
         env:
           JACOCO_COVERAGE: true
           WHITESOURCE_API_KEY: ${{ secrets.WHITESOURCE_API_KEY }}
@@ -37,6 +40,7 @@ jobs:
       - name: Code Coverage
         uses: codecov/codecov-action@v1
   build-windows:
+    name: Build & Test - Windows, JDK ${{ matrix.java }}, Cloud AWS
     runs-on: windows-2022
     strategy:
       matrix:
diff --git a/profile_azure.json.gpg b/profile_azure.json.gpg
new file mode 100644
index 0000000000000000000000000000000000000000..b388626dce176b3d587246bc4c8f9e88cdb9f2be
GIT binary patch
literal 1593
zcmV-92FCe}4Fm}T0*51Db!f;jsp-<{0Xmdh$|5ZWjI|JiYM(gAf2HUw3TyR3|90Ra
zO0|YxVDpP|)?qK~!%A<igdpuTQy6DS4y7t-cyyT;AA?JFJ5a(5?K>tkS9A96LiRoV
z{AQLg)v#!Ra^+w^3e;U$3(=FzBV8Idqm1VgvJUerVCVSHdc4!+b%I4U;cd){UNvz$
zsu0c9Ppd-=$0T~d4K{6zObR+fN3{U8pP52E7pv?D%U(u$vLNCQM*+C!_9T$_k9;_f
z-}CP9%I*AS_7vBAlq42Y3t*b_x8^y%o%8%Q>fj!6Rdk+sZ@%RoFPmjZv}*H^3JPZZ
zQ<CyfOua44)B8_zQj)w@4xL*-1lUBLn!XbE?Y{T*fY3`YQq?X329Gg&>O;8SWS=h0
zsELgQC`#A4X_X}p9r=LbYdm5olH=u(yT5Bo?|TC^UL9of2LLN1bZ3nGRWk1jW)W5|
zaEWr)N$x5Z_S41hb5k5&_i!4)?%KLz-v}kPCe0;<-ofN~NzixLj6FwKlTn+C6#w?}
zmJ2v9o5q{#N>kBPEOY^H<}MW7+NkBw$rf%VGKqvu`J>}OF*)XN__+=Aqq_OfZ^|&0
zl$H{AzuyQ)fu{SG&Mxl-$}is<hnuL4qflv2VJE4E%VYQu_x-<@$g)pACX$bMA22d+
ztv+W_hd#i0TqoEZoRaG4o+1vw&<`4`L+NENR3D47H}1t{-bxC*$$<W1N~)P=#s{j@
zD2^(~48iRp9_&hw>ovnC8!#I|HD8v6iE=~OvSwV!EbA-|;@?~L+`{?hi0XY5w#;vb
zf8nrp@$6D*{-JXHjD)5eYq(kGeQy6WNl{u!>cbP&NX*3GL>Zll=|~lMW;;xop<!g4
zD5oqZ|L$Bi@G{oN*j^<Z7_I>te*^ijbPkzJs)IwrJYzA@f@ZJsw?AoWHq^lzuAB`(
z?#%R0_JplYkfT{-AJ7EeNQ(V_;*`z}_+kBXOm~Ru`#vw7I>j%Tqj{yPS%fz}!;BKp
zG`nc>wbTkWU8m3Fk1gi<ZH>vQdC~?K9!lg#8!D28fM@qxQr6(G^E+jFacA5LV~c||
z?trp7#2EmEA%mbDi9^ljLE2FQH_K>HMpVgW&sgD1^MbC^eCs(O+uG&N+L^G@A0Atr
zk-d37N5~U*0x(<Ms-{<rD7cLePL@myf&d}kH<R8|Arw1aYeMP9`JiKcn@HvwkXX@r
zP1Rw!@97G*QQ(&GnKX}6Vy-#-MkhsGa3UWS%-7<zW?ToCh+VO@r)JwY7R35u;G{J%
zeV8#=h1PSs2}J5@gp3)XG14opt&)}s#qO)j-O$uV&1ZAo=p(a`RaB%|o}KHZO8GN>
zDvhJ-#GcAK>3Q>IA0bbvXwLiqq885Es6S#1+q^kTHeWh_&gJR>MV&)CnMcc0qP4>g
z3Zp7uTb>1dbnjs3Pat=q^()B}a1)tHSqT2J0f+s-*y;>hWC$bw0N1eZJgMrD+!K2!
zvKMBJt<ysEqMa*XHeI$0N`6D47~ucyl-6t^s_m28K>ZK=X5M1S#lv~=@N&qwn_*B{
zDeZlNVTCj%IMZQ0J(~BFFnNwi3h{eR>AU?Fp(qxA4Q(P&BQ>GdJfFwnLLIpV3}m~E
zoX>GW6%r$4fnXy1a@}_Q-(I(LFRy=paYHuI{|s$xw}Mbk*Ts&P%Zvattdx6rhowWq
zLD+_0pah|B``Y!-$$><cMZOKiA|mGFgi~7xo5O;#^&}2xMivHBVeFm5Us21Djs+OT
zjU|r{pC5$XpPy^V6@Z8i`#*Pc0(85UY*;H)L8Fuo_@My8hK1a4J4@V<J&ja;s+p)B
zw&Nz7GmA}4Oi_poWx$2Hh0VE1rk9lq2hl@Eg}ep!xSsfV%$(rzB&B~N0dBZt#IiY6
zE}^w;UJ;P|y&bN>vo1$b7&t9O+=@e`^x)Pkj`weBJlFbtaV;5{qQb4Bfd%-lT8?^8
z+bej$gbGOFVDW97(8b$UpTcLyG-UZr75Z3{1LQ?~a5`+(QYOoQR+J9B8&AzKyDl*{
rza~I8Wc?5BzT-vEQOLaQE3nI1kbu&IERznw(iAPCgIWTW_U6diAE6vQ

literal 0
HcmV?d00001

diff --git a/profile_gcp.json.gpg b/profile_gcp.json.gpg
new file mode 100644
index 0000000000000000000000000000000000000000..cc45b109f82382db03b6abd53593bac7fbb727a9
GIT binary patch
literal 1594
zcmV-A2F3Y|4Fm}T0u)pK1n}+^a_Q3Q0r0~mG~W=w0}GeyToL{cNvHE|3<#{KBG7Od
zvx`Ad>C!8?BVagFVoh0db84|TM9pz=!?cq13|CBg8<B--I+N1ZV%=_zqc@jyn05P1
z2^Sv*zBLRgU-Z^#z|G>h*Un@;8=S$C6}<4kJprFxq~yiR+ht>_JMSV#G-yjh#F&M0
z?eeq7mWt}`0%h<aRjn!fP*XgVWC%Dgq9Gc#vw-B8jPP3@UsD16y*}&Y<xPx$Xm5aJ
z=EkoYOcb~bftM*>u!j<{A<-F0bH)K6=`7)BTyFF-Pi*et1!iq$8XS#*!mZek^#9%5
z2-c7(a?Yt7ROi*4_Hi#lOe^ezWp0dEm#s01!?HyTP(zqJdL|jELhpa}TdwG$@GwiT
zjA*xZyK65!-M{^MguoxYfLqCorfS-z4dLYI6tje&ln7FUci*Zd5X~DZA)OT`-SYRL
z9gq?@n~K%oQc=Opkw5Ny!gSi>p}`itwm3}>TG>?p(vk5cUru-K4PAOpab^HXg1XaI
zSc1`Cb`K=B=vh?g#0`Q~YS5GZO)CE`G_Ci+<@`G6OPHy`7|(JpRgo2@3HYQ3q|amS
z%pxwZP;x@DkybEvT)oL|OTQxhLs#3crjK*ZGyB#@8J<dUJAdG}_5k0l1|`>}*;kN#
zh#H0;Hlcn^KOrrasRmoprrkPsOn#qNYOO>RtItAv;K-I%D(sw8`SQVN8)U7~{21Ln
z{_G%G^z_Cn>}^}232qsTjMj8x8MFE6D1Z@82vTZ8T(+LE%CX*@Ri1?Qd$xfq3}&|O
zoTvVE(7V~=z&nKvpooj>7RlLiN;Rb48z(5}Z+rb95na98I~4QO3sf(o>fY;d4KUh!
zh7;Uw2disxz%iaXeNn?q1Psf9i`g3!BPS}85a6>bQ*9w(&r1y3eqkv3Ic(!mol<Ef
zJ8;zk!Ho_g@3aCdYGEC@-8fK;M2h_JpaDZg*WlgiA;e({wDjyDUKakhsMy~Pb%92T
zWD*K<?fP7Bw80o~wdqz3#cvc|Z1^_KRJoYs$6iwQrHLPn=QGEYi%#+>rboY)X4xBe
z>x={nF5+UFB2Y2y4s31~jr0r`F!EA#?wc>m3^2b}D9zE96f&^wu1-YkzVgjr3CGFw
zpU(KMSc0|;x1n1M``NM=tGSLTV#GKu=#PrOb(x#DubFv*m4Vn&EEm{8bCx<uw_cmT
zKc!NHBxllYWw4mLAdOAVIiGpGbsYCck$7QCE_y30IBL4Whr6||3?vvw15Hq+<eT^J
z+%gd;*|0&%h2zh}iDQD?vL4Y53d_q#*<kg!lIQj|6$gw$Ea3W@1PM&pbrXvSVm9Z2
zP6g%8Mh;Z<>0yqPl%t9!P4Cgwfdc#U43MUZ`gb`oRH&Ng(J6*cFn2a|<gDHEw%OaS
z@DK)U!BxYJbotkoI%<#N^e*q<F!7j8mqFt~2zbRnh$(zAsRYiL)7ax$yKVzcP$r5^
zKFbWs*X;(*7AzctIjIHv)ZkSPi%60#xax?#{oSx68ABL|VRk%C$4ykjztVabn!}1m
z@r`fE9rWpiE?LTj;|cw+{;oS`l7F?@>b_T++A+h}l|_r)9x>C_cso~>V7rgWv0d6Y
zCH$F-<PDnVR}zS4&K~pMpLX{FblO>$7G6`_L1q)gTx}C~@%CLwER$^5*L%ixkjK0%
zYA%8r7I9tdC~I=Dt(gPkh)e?UaUnmJfd-O}N<=m)iu{FbS!MC^Gi`TZ7Rt&j`jLFx
zHDjvIlflMS*;!FrUa7$JVb;u!W<^>QCI^dVHkaI{w=)1KDDh1*Pf6h+Z#}R^4T>Gj
zAmpnkK)F{K4q{_Yu*JZo#o9*e3HmhtE}=pMb$p&JVEF1DZJqS?_p1@T7UFSw7wF$7
zc0IAKzcl!6bwVbQhqqH&lsc?AYOCNajxvI`I-gOBTce_YMqDZl2`#I^si0)Ulte)l
zjBxTZ@!!bBD^=P=d-((++o(Y&TmjUrL{JVY?oRlOp{Cxo{FvxkPdb6IkO5tt-G5!S
sk|*3Rtr}laMms=KHn_L?LNG(pk1BH+f+BTKHhO39q^?}NG*_(>+g3Iko&W#<

literal 0
HcmV?d00001

diff --git a/scripts/decrypt_secret.sh b/scripts/decrypt_secret.sh
index 05df864e6..37b923ff2 100755
--- a/scripts/decrypt_secret.sh
+++ b/scripts/decrypt_secret.sh
@@ -3,5 +3,21 @@
 # Decrypt the file
 # --batch to prevent interactive command --yes to assume "yes" for questions
 
-gpg --quiet --batch --yes --decrypt --passphrase="$DECRYPTION_PASSPHRASE" \
-  --output profile.json ./profile.json.gpg
\ No newline at end of file
+# Input one argument
+snowflake_deployment=$1
+
+# Convert the input to uppercase
+uppercase_deployment=$(echo $snowflake_deployment | tr '[:lower:]' '[:upper:]')
+
+if [ $uppercase_deployment = 'AWS' ]; then
+  gpg --quiet --batch --yes --decrypt --passphrase="$DECRYPTION_PASSPHRASE" \
+  --output profile.json ./profile.json.gpg
+elif [ $uppercase_deployment = 'GCP' ]; then
+  gpg --quiet --batch --yes --decrypt --passphrase="$DECRYPTION_PASSPHRASE" \
+    --output profile.json ./profile_gcp.json.gpg
+elif [ $uppercase_deployment = 'AZURE' ]; then
+  gpg --quiet --batch --yes --decrypt --passphrase="$DECRYPTION_PASSPHRASE" \
+    --output profile.json ./profile_azure.json.gpg
+else
+  echo "Invalid deployment option. Please enter 'AWS', 'AZURE', or 'GCP'."
+fi
\ No newline at end of file

From 2f6e8afe7ef01d41a7c48e7c4d11c11d181b6c54 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Fri, 21 Apr 2023 13:51:26 -0700
Subject: [PATCH 198/356] V1.1.2 release (#482)

v1.1.2 release
---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index a5ed47d3a..365f732c1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>1.1.1</version>
+  <version>1.1.2</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 746f8122d..7d162b1f1 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -127,7 +127,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.1.1";
+  public static final String DEFAULT_VERSION = "1.1.2";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 0c06ac1c2ad594366ce41a8c48aaa5808e26653c Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Mon, 24 Apr 2023 17:28:15 +0200
Subject: [PATCH 199/356] NO-SNOW Fix flaky Windows test (#484)

The test `SnowflakeStreamingIngestChannelTest#testInsertRowThrottling` is
flaky on Windows. The reason is that we are using mockito to mock methods
from java.lang.Runtime in one thread and read them in another. These
changes are not always guaranteed to be visible in the other thread.

This PR introduces a new interface with two implementations to provide
memory info, so we don't need to mock anything anymore.
---
 .../internal/MemoryInfoProvider.java          | 17 ++++++++
 .../MemoryInfoProviderFromRuntime.java        | 23 ++++++++++
 ...owflakeStreamingIngestChannelInternal.java | 14 ++++---
 .../SnowflakeStreamingIngestChannelTest.java  | 42 +++++++++++++++----
 4 files changed, 81 insertions(+), 15 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/MemoryInfoProvider.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/MemoryInfoProviderFromRuntime.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/MemoryInfoProvider.java b/src/main/java/net/snowflake/ingest/streaming/internal/MemoryInfoProvider.java
new file mode 100644
index 000000000..f426e898d
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/MemoryInfoProvider.java
@@ -0,0 +1,17 @@
+/*
+ * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+/** Provider information about available system memory */
+public interface MemoryInfoProvider {
+  /** @return Max memory the JVM can allocate */
+  long getMaxMemory();
+
+  /** @return Total allocated JVM memory so far */
+  long getTotalMemory();
+
+  /** @return Free JVM memory */
+  long getFreeMemory();
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/MemoryInfoProviderFromRuntime.java b/src/main/java/net/snowflake/ingest/streaming/internal/MemoryInfoProviderFromRuntime.java
new file mode 100644
index 000000000..3a957f225
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/MemoryInfoProviderFromRuntime.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+/** Reads memory information from JVM runtime */
+public class MemoryInfoProviderFromRuntime implements MemoryInfoProvider {
+  @Override
+  public long getMaxMemory() {
+    return Runtime.getRuntime().maxMemory();
+  }
+
+  @Override
+  public long getTotalMemory() {
+    return Runtime.getRuntime().totalMemory();
+  }
+
+  @Override
+  public long getFreeMemory() {
+    return Runtime.getRuntime().freeMemory();
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index f1a4da9d3..3916bc0d9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -346,7 +346,7 @@ public InsertValidationResponse insertRow(Map<String, Object> row, String offset
   @Override
   public InsertValidationResponse insertRows(
       Iterable<Map<String, Object>> rows, String offsetToken) {
-    throttleInsertIfNeeded(Runtime.getRuntime());
+    throttleInsertIfNeeded(new MemoryInfoProviderFromRuntime());
     checkValidation();
 
     if (isClosed()) {
@@ -399,11 +399,11 @@ public String getLatestCommittedOffsetToken() {
   }
 
   /** Check whether we need to throttle the insertRows API */
-  void throttleInsertIfNeeded(Runtime runtime) {
+  void throttleInsertIfNeeded(MemoryInfoProvider memoryInfoProvider) {
     int retry = 0;
     long insertThrottleIntervalInMs =
         this.owningClient.getParameterProvider().getInsertThrottleIntervalInMs();
-    while ((hasLowRuntimeMemory(runtime)
+    while ((hasLowRuntimeMemory(memoryInfoProvider)
             || (this.owningClient.getFlushService() != null
                 && this.owningClient.getFlushService().throttleDueToQueuedFlushTasks()))
         && retry < INSERT_THROTTLE_MAX_RETRY_COUNT) {
@@ -425,7 +425,7 @@ void throttleInsertIfNeeded(Runtime runtime) {
   }
 
   /** Check whether we have a low runtime memory condition */
-  private boolean hasLowRuntimeMemory(Runtime runtime) {
+  private boolean hasLowRuntimeMemory(MemoryInfoProvider memoryInfoProvider) {
     int insertThrottleThresholdInBytes =
         this.owningClient.getParameterProvider().getInsertThrottleThresholdInBytes();
     int insertThrottleThresholdInPercentage =
@@ -434,9 +434,11 @@ private boolean hasLowRuntimeMemory(Runtime runtime) {
         this.owningClient.getParameterProvider().getMaxMemoryLimitInBytes();
     long maxMemory =
         maxMemoryLimitInBytes == MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT
-            ? runtime.maxMemory()
+            ? memoryInfoProvider.getMaxMemory()
             : maxMemoryLimitInBytes;
-    long freeMemory = runtime.freeMemory() + (runtime.maxMemory() - runtime.totalMemory());
+    long freeMemory =
+        memoryInfoProvider.getFreeMemory()
+            + (memoryInfoProvider.getMaxMemory() - memoryInfoProvider.getTotalMemory());
     boolean hasLowRuntimeMemory =
         freeMemory < insertThrottleThresholdInBytes
             && freeMemory * 100 / maxMemory < insertThrottleThresholdInPercentage;
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 5e593db05..d34386bfa 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -45,6 +45,30 @@
 
 public class SnowflakeStreamingIngestChannelTest {
 
+  /**
+   * Mock memory provider, allows the user to set what memory values should be returned. Fields are
+   * volatile, so can be modified by another threads than the one reading it.
+   */
+  private static class MockedMemoryInfoProvider implements MemoryInfoProvider {
+    public volatile long freeMemory;
+    public volatile long maxMemory;
+
+    @Override
+    public long getMaxMemory() {
+      return maxMemory;
+    }
+
+    @Override
+    public long getTotalMemory() {
+      return maxMemory;
+    }
+
+    @Override
+    public long getFreeMemory() {
+      return freeMemory;
+    }
+  }
+
   @Test
   public void testChannelFactoryNullFields() {
     String name = "CHANNEL";
@@ -539,7 +563,11 @@ public void testInsertRow() {
 
   @Test
   public void testInsertRowThrottling() {
-    long maxMemory = 1000000L;
+    final long maxMemory = 1000000L;
+
+    final MockedMemoryInfoProvider memoryInfoProvider = new MockedMemoryInfoProvider();
+    memoryInfoProvider.maxMemory = maxMemory;
+
     SnowflakeStreamingIngestClientInternal<?> client =
         new SnowflakeStreamingIngestClientInternal<>("client");
     SnowflakeStreamingIngestChannelInternal<?> channel =
@@ -557,16 +585,12 @@ public void testInsertRowThrottling() {
             OpenChannelRequest.OnErrorOption.CONTINUE,
             UTC);
 
-    Runtime mockedRunTime = Mockito.mock(Runtime.class);
-    Mockito.when(mockedRunTime.maxMemory()).thenReturn(maxMemory);
-    Mockito.when(mockedRunTime.totalMemory()).thenReturn(maxMemory);
     ParameterProvider parameterProvider = new ParameterProvider();
-    Mockito.when(mockedRunTime.freeMemory())
-        .thenReturn(
-            maxMemory * (parameterProvider.getInsertThrottleThresholdInPercentage() - 1) / 100);
+    memoryInfoProvider.freeMemory =
+        maxMemory * (parameterProvider.getInsertThrottleThresholdInPercentage() - 1) / 100;
 
     CompletableFuture<Void> future =
-        CompletableFuture.runAsync(() -> channel.throttleInsertIfNeeded(mockedRunTime));
+        CompletableFuture.runAsync(() -> channel.throttleInsertIfNeeded(memoryInfoProvider));
 
     try {
       future.get(5L, TimeUnit.SECONDS);
@@ -577,7 +601,7 @@ public void testInsertRowThrottling() {
       Assert.fail("unexpected exception encountered.");
     }
 
-    Mockito.when(mockedRunTime.freeMemory()).thenReturn(1000000L);
+    memoryInfoProvider.freeMemory = maxMemory;
 
     // We should succeed now
     try {

From 40e49d081530bb94f23ed1bb9c3a1f74adce8a3e Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Mon, 24 Apr 2023 15:57:22 -0700
Subject: [PATCH 200/356] =?UTF-8?q?Revert=20"SNOW-787565:=20Use=20dynamic?=
 =?UTF-8?q?=20scaling=20thread=20pool=20instead=20of=20fixed=E2=80=A6=20(#?=
 =?UTF-8?q?486)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit eb51202 since it's causing a lot more channel invalidations during low internet cases
---
 .../ingest/streaming/internal/FlushService.java          | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 0765ed230..f738d33be 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -31,7 +31,6 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -321,13 +320,7 @@ private void createWorkers() {
                 * (1 + this.owningClient.getParameterProvider().getIOTimeCpuRatio()),
             MAX_THREAD_COUNT);
     this.buildUploadWorkers =
-        new ThreadPoolExecutor(
-            1,
-            buildUploadThreadCount,
-            60L,
-            TimeUnit.SECONDS,
-            new SynchronousQueue<Runnable>(),
-            buildUploadThreadFactory);
+        Executors.newFixedThreadPool(buildUploadThreadCount, buildUploadThreadFactory);
 
     logger.logInfo(
         "Create {} threads for build/upload blobs for client={}, total available processors={}",

From 8503387af5dbc1d5cc8740210749a011ab0e4ebf Mon Sep 17 00:00:00 2001
From: Jay Patel <jay.patel@snowflake.com>
Date: Tue, 25 Apr 2023 00:00:41 -0700
Subject: [PATCH 201/356] Enable AZURE and GCP tests to run on windows machine
 (#483)

---
 .github/workflows/End2EndTest.yml  | 10 +++++++---
 scripts/decrypt_secret_windows.ps1 | 25 +++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 3 deletions(-)
 create mode 100644 scripts/decrypt_secret_windows.ps1

diff --git a/.github/workflows/End2EndTest.yml b/.github/workflows/End2EndTest.yml
index be7f13442..12b144c76 100644
--- a/.github/workflows/End2EndTest.yml
+++ b/.github/workflows/End2EndTest.yml
@@ -40,11 +40,13 @@ jobs:
       - name: Code Coverage
         uses: codecov/codecov-action@v1
   build-windows:
-    name: Build & Test - Windows, JDK ${{ matrix.java }}, Cloud AWS
+    name: Build & Test - Windows, JDK ${{ matrix.java }}, Cloud ${{ matrix.snowflake_cloud }}
     runs-on: windows-2022
     strategy:
+      fail-fast: false
       matrix:
         java: [ 8 ]
+        snowflake_cloud: [ 'AWS', 'AZURE', 'GCP' ]
     steps:
       - name: Checkout Code
         uses: actions/checkout@v2
@@ -52,10 +54,12 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: ${{ matrix.java }}
-      - name: Decrypt profile.json (Windows)
+      - name: Decrypt profile.json for Cloud ${{ matrix.snowflake_cloud }} on Windows Powershell
         env:
           DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }}
-        run: gpg --quiet --batch --yes --decrypt --passphrase="${env:DECRYPTION_PASSPHRASE}" --output profile.json ./profile.json.gpg
+        shell: pwsh
+        run: |
+          ./scripts/decrypt_secret_windows.ps1 -SnowflakeDeployment '${{ matrix.snowflake_cloud }}'
       - name: Unit & Integration Test (Windows)
         continue-on-error: false
         run: mvn -DghActionsIT verify --batch-mode
diff --git a/scripts/decrypt_secret_windows.ps1 b/scripts/decrypt_secret_windows.ps1
new file mode 100644
index 000000000..e0cd0b589
--- /dev/null
+++ b/scripts/decrypt_secret_windows.ps1
@@ -0,0 +1,25 @@
+param(
+    [string]$SnowflakeDeployment
+)
+
+# Convert the input to uppercase
+$UppercaseDeployment = $SnowflakeDeployment.ToUpper()
+
+# Decrypt the file based on the deployment option
+switch ($UppercaseDeployment) {
+    'AWS' {
+        gpg --quiet --batch --yes --decrypt --passphrase="$env:DECRYPTION_PASSPHRASE" `
+            --output profile.json ./profile.json.gpg
+    }
+    'GCP' {
+        gpg --quiet --batch --yes --decrypt --passphrase="$env:DECRYPTION_PASSPHRASE" `
+            --output profile.json ./profile_gcp.json.gpg
+    }
+    'AZURE' {
+        gpg --quiet --batch --yes --decrypt --passphrase="$env:DECRYPTION_PASSPHRASE" `
+            --output profile.json ./profile_azure.json.gpg
+    }
+    default {
+        Write-Output "Invalid deployment option. Please enter 'AWS', 'AZURE', or 'GCP'."
+    }
+}

From 2714e9bb121ad6b8d37d75a7a7df6a2901369c8d Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Tue, 25 Apr 2023 10:38:14 -0700
Subject: [PATCH 202/356] V1.1.3 release (#487)

Release 1.1.3 which reverts a regression in 1.1.2
---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 365f732c1..d75337d61 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>1.1.2</version>
+  <version>1.1.3</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 7d162b1f1..b3b2b5947 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -127,7 +127,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.1.2";
+  public static final String DEFAULT_VERSION = "1.1.3";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 08321e5f76001a2e461c85fac4965c1e1469567a Mon Sep 17 00:00:00 2001
From: Hongke Qin <hongke.qin@snowflake.com>
Date: Fri, 28 Apr 2023 18:23:38 -0700
Subject: [PATCH 203/356] Adding a script to release unshaded version into S3
 repo (#346)

* Adding a script to deploy unshaded version of the SDK

* Add Changes for unshaded release

* Skip Docs Generation for snowflake ingest java

* Enable Dropping

* removing javadoc and sources jar for testing

* Unshaded Release

* Undo Public Pom.xml Version

* Update for 1.1.3 unshaded release

---------

Co-authored-by: sfc-gh-kdama <kirankumar.dama@snowflake.com>
Co-authored-by: David Wang <david.wang@snowflake.com>
Co-authored-by: David Wang <63617111+sfc-gh-dwang@users.noreply.github.com>
---
 scripts/update_project_version.py |  6 +++
 unshaded_deploy.sh                | 88 +++++++++++++++++++++++++++++++
 2 files changed, 94 insertions(+)
 create mode 100755 unshaded_deploy.sh

diff --git a/scripts/update_project_version.py b/scripts/update_project_version.py
index 6ed3d47c1..dd844a7c5 100755
--- a/scripts/update_project_version.py
+++ b/scripts/update_project_version.py
@@ -5,6 +5,7 @@
 #
 # arg1: pom.xml
 # arg2: new version number
+# arg3: optional suffix for project name
 #
 import sys
 import xml.etree.ElementTree as ET
@@ -12,6 +13,9 @@
 
 pom_file = sys.argv[1]
 version = sys.argv[2]
+suffix = None
+if len(sys.argv) > 3:
+    suffix = sys.argv[3]
 
 def remove_namespace(doc, namespace):
     """
@@ -42,6 +46,8 @@ def handle_comment(self, data):
 for e1 in root:
     if type(e1.tag) is str and e1.tag.endswith('version'):
         e1.text = version
+    if suffix and type(e1.tag) is str and e1.tag.endswith('artifactId'):
+        e1.text = e1.text + '-' + suffix
 
 remove_namespace(root, u'http://maven.apache.org/POM/4.0.0')
 
diff --git a/unshaded_deploy.sh b/unshaded_deploy.sh
new file mode 100755
index 000000000..a5e607291
--- /dev/null
+++ b/unshaded_deploy.sh
@@ -0,0 +1,88 @@
+#!/bin/bash -e
+
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+export GPG_KEY_ID="Snowflake Computing"
+export SONATYPE_USER="$sonatype_user"
+export SONATYPE_PWD="$sonatype_password"
+
+if [ -z "$GPG_KEY_PASSPHRASE" ]; then
+  echo "[ERROR] GPG passphrase is not specified for $GPG_KEY_ID!"
+  exit 1
+fi
+
+if [ -z "$GPG_PRIVATE_KEY" ]; then
+  echo "[ERROR] GPG private key file is not specified!"
+  exit 1
+fi
+
+echo "[INFO] Import PGP Key"
+if ! gpg --list-secret-key | grep "$GPG_KEY_ID"; then
+  gpg --allow-secret-key-import --import "$GPG_PRIVATE_KEY"
+fi
+
+# copy the settings.xml template and inject credential information
+OSSRH_DEPLOY_SETTINGS_XML="$THIS_DIR/mvn_settings_ossrh_deploy.xml"
+MVN_REPOSITORY_ID=ossrh
+
+cat > $OSSRH_DEPLOY_SETTINGS_XML << SETTINGS.XML
+<?xml version="1.0" encoding="UTF-8"?>
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+     xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
+  <servers>
+    <server>
+      <id>$MVN_REPOSITORY_ID</id>
+      <username>$SONATYPE_USER</username>
+      <password>$SONATYPE_PWD</password>
+    </server>
+  </servers>
+</settings>
+SETTINGS.XML
+
+MVN_OPTIONS+=(
+  "--settings" "$OSSRH_DEPLOY_SETTINGS_XML"
+  "--batch-mode"
+)
+
+echo "[Info] Sign unshaded package and deploy to staging area"
+project_version=$($THIS_DIR/scripts/get_project_info_from_pom.py $THIS_DIR/pom.xml version)
+echo "[Info] Project version: $project_version"
+$THIS_DIR/scripts/update_project_version.py pom.xml ${project_version} > generated_public_pom.xml
+
+mvn deploy ${MVN_OPTIONS[@]} -Dnot-shadeDep -Dossrh-deploy 
+
+echo "[INFO] Close and Release"
+snowflake_repositories=$(mvn ${MVN_OPTIONS[@]} \
+    org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-list \
+    -DserverId=$MVN_REPOSITORY_ID  -Dnot-shadeDep  versions:set -DnewVersion=$project_version-unshaded -Dmaven.javadoc.skip=true -Dmaven.source.skip=true \
+    -DnexusUrl=https://oss.sonatype.org/ | grep netsnowflake | awk '{print $2}')
+IFS=" "
+if (( $(echo $snowflake_repositories | wc -l)!=1 )); then
+    echo "[ERROR] Not single netsnowflake repository is staged. Login https://oss.sonatype.org/ and make sure no netsnowflake remains there."
+    exit 1
+fi
+if ! mvn ${MVN_OPTIONS[@]} \
+    org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-close \
+    -DserverId=$MVN_REPOSITORY_ID \
+    -DnexusUrl=https://oss.sonatype.org/ \
+    -DstagingRepositoryId=$snowflake_repositories \
+    -Dnot-shadeDep  versions:set -DnewVersion=$project_version-unshaded -Dmaven.javadoc.skip=true -Dmaven.source.skip=true\
+    -DstagingDescription="Automated Close"; then
+    echo "[ERROR] Failed to close. Fix the errors and try this script again"
+    mvn ${MVN_OPTIONS[@]} \
+        nexus-staging:rc-drop \
+        -DserverId=$MVN_REPOSITORY_ID \
+        -DnexusUrl=https://oss.sonatype.org/ \
+        -DstagingRepositoryId=$snowflake_repositories -Dnot-shadeDep  versions:set -DnewVersion=$project_version-unshaded -Dmaven.javadoc.skip=true -Dmaven.source.skip=true\
+        -DstagingDescription="Failed to close. Dropping..."
+fi
+
+mvn ${MVN_OPTIONS[@]} \
+    org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-release \
+    -DserverId=$MVN_REPOSITORY_ID \
+    -DnexusUrl=https://oss.sonatype.org/ \
+    -DstagingRepositoryId=$snowflake_repositories -Dnot-shadeDep  versions:set -DnewVersion=$project_version-unshaded -Dmaven.javadoc.skip=true -Dmaven.source.skip=true\
+    -DstagingDescription="Automated Release"
+
+rm $OSSRH_DEPLOY_SETTINGS_XML

From 83c59134eda8d4b7f80388de6dd081e4f0ba8f74 Mon Sep 17 00:00:00 2001
From: David Wang <63617111+sfc-gh-dwang@users.noreply.github.com>
Date: Thu, 4 May 2023 00:40:13 -0700
Subject: [PATCH 204/356] Update protobuf version to 3.19.6 (#488)

Update protobuf version
---
 pom.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/pom.xml b/pom.xml
index d75337d61..3790bc8a3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,6 +57,7 @@
     <objenesis.version>3.1</objenesis.version>
     <parquet.version>1.12.3</parquet.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <protobuf.version>3.19.6</protobuf.version>
     <shadeBase>net.snowflake.ingest.internal</shadeBase>
     <slf4j.version>1.7.36</slf4j.version>
     <snappy.version>1.1.8.3</snappy.version>
@@ -78,6 +79,11 @@
         <artifactId>guava</artifactId>
         <version>${guava.version}</version>
       </dependency>
+      <dependency>
+        <groupId>com.google.protobuf</groupId>
+        <artifactId>protobuf-java</artifactId>
+        <version>${protobuf.version}</version>
+      </dependency>
       <dependency>
         <groupId>com.nimbusds</groupId>
         <artifactId>nimbus-jose-jwt</artifactId>

From 5cc2e7f3f1e6669d4f957860e0955fe4ec37f661 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Fri, 5 May 2023 00:12:36 +0200
Subject: [PATCH 205/356] SNOW-807102 Disable OpenManyChannelsIT (#491)

---
 .../snowflake/ingest/streaming/internal/OpenManyChannelsIT.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/OpenManyChannelsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/OpenManyChannelsIT.java
index 006623728..c3752a660 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/OpenManyChannelsIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/OpenManyChannelsIT.java
@@ -23,9 +23,11 @@
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
 /** Tries to open several thousand channels into the same table from multiple threads in parallel */
+@Ignore("Will be reimplemented in dew: SNOW-807102")
 public class OpenManyChannelsIT {
   private static final int THREAD_COUNT = 20;
   private static final int CHANNELS_PER_THREAD = 250;

From a8f9275f65eafe0ea1a8f5bd978d28315b6eb10a Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Fri, 5 May 2023 19:21:14 +0200
Subject: [PATCH 206/356] SNOW-801398 Tidy up shaded jar (#490)

---
 pom.xml                   | 142 ++++++++++++++++++++++++++++++++++++--
 scripts/check_content.sh  |  55 ++++++++-------
 scripts/run_gh_actions.sh |   5 ++
 3 files changed, 173 insertions(+), 29 deletions(-)

diff --git a/pom.xml b/pom.xml
index 3790bc8a3..196375b3b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -153,18 +153,70 @@
             <groupId>ch.qos.reload4j</groupId>
             <artifactId>reload4j</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>com.github.pjfanning</groupId>
+            <artifactId>jersey-json</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>com.sun.jersey</groupId>
+            <artifactId>jersey-core</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>com.sun.jersey</groupId>
+            <artifactId>jersey-server</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>com.sun.jersey</groupId>
+            <artifactId>jersey-servlet</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>dnsjava</groupId>
+            <artifactId>dnsjava</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>javax.activation</groupId>
             <artifactId>activation</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>javax.servlet</groupId>
+            <artifactId>javax.servlet-api</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>javax.servlet.jsp</groupId>
+            <artifactId>jsp-api</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.avro</groupId>
+            <artifactId>avro</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-auth</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.kerby</groupId>
+            <artifactId>kerb-core</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>org.apache.zookeeper</groupId>
             <artifactId>zookeeper</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.eclipse.jetty</groupId>
+            <artifactId>jetty-server</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.eclipse.jetty</groupId>
+            <artifactId>jetty-servlet</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.eclipse.jetty</groupId>
+            <artifactId>jetty-util</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.eclipse.jetty</groupId>
+            <artifactId>jetty-webapp</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-log4j12</artifactId>
@@ -205,11 +257,6 @@
         <artifactId>jackson-jaxrs</artifactId>
         <version>${codehaus.version}</version>
       </dependency>
-      <dependency>
-        <groupId>org.codehaus.jackson</groupId>
-        <artifactId>jackson-mapper-asl</artifactId>
-        <version>${codehaus.version}</version>
-      </dependency>
       <dependency>
         <groupId>org.codehaus.jackson</groupId>
         <artifactId>jackson-xc</artifactId>
@@ -697,8 +744,13 @@
       <id>checkShadedContent</id>
       <activation>
         <os>
+          <!-- Shaded JAR content checking is a shell script, do not run it on Windows -->
           <family>!Windows</family>
         </os>
+        <property>
+          <!-- Do not check shaded JAR content if shading has been disabled -->
+          <name>!not-shadeDep</name>
+        </property>
       </activation>
       <build>
         <plugins>
@@ -736,6 +788,11 @@
             <artifactId>maven-shade-plugin</artifactId>
             <version>3.4.1</version>
             <configuration>
+              <artifactSet>
+                <excludes>
+                  <exclude>net.snowflake:snowflake-jdbc</exclude>
+                </excludes>
+              </artifactSet>
               <relocations>
                 <relocation>
                   <pattern>com.nimbusds</pattern>
@@ -761,11 +818,85 @@
                   <pattern>org.apache</pattern>
                   <shadedPattern>${shadeBase}.apache</shadedPattern>
                 </relocation>
+                <relocation>
+                  <pattern>org.xbill</pattern>
+                  <shadedPattern>${shadeBase}.org.xbill</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.xerial</pattern>
+                  <shadedPattern>${shadeBase}.org.xerial</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>io.netty</pattern>
+                  <shadedPattern>${shadeBase}.io.netty</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>com.google</pattern>
+                  <shadedPattern>${shadeBase}.com.google</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>com.github.benmanes</pattern>
+                  <shadedPattern>${shadeBase}.com.github.benmanes</shadedPattern>
+                </relocation>
+                <!-- shaded.parquet contains dependencies shaded in parquet itself. Here we are shading it for the second time -->
+                <relocation>
+                  <pattern>shaded.parquet</pattern>
+                  <shadedPattern>${shadeBase}.shaded.parquet</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.codehaus</pattern>
+                  <shadedPattern>${shadeBase}.org.codehaus</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>com.jcraft</pattern>
+                  <shadedPattern>${shadeBase}.com.jcraft</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.eclipse</pattern>
+                  <shadedPattern>${shadeBase}.org.eclipse</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>org.checkerframework</pattern>
+                  <shadedPattern>${shadeBase}.org.checkerframework</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>com.codahale</pattern>
+                  <shadedPattern>${shadeBase}.com.codahale</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>com.ctc</pattern>
+                  <shadedPattern>${shadeBase}.com.ctc</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>com.github.luben</pattern>
+                  <shadedPattern>${shadeBase}.com.github.luben</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>com.thoughtworks</pattern>
+                  <shadedPattern>${shadeBase}.com.thoughtworks</shadedPattern>
+                </relocation>
+                <!-- This belongs to org.apache.arrow:arrow-vector -->
+                <relocation>
+                  <pattern>codegen</pattern>
+                  <shadedPattern>${shadeBase}.codegen</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>javax.annotation</pattern>
+                  <shadedPattern>${shadeBase}.javax.annotation</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>javax.activation</pattern>
+                  <shadedPattern>${shadeBase}.javax.activation</shadedPattern>
+                </relocation>
               </relocations>
               <filters>
                 <filter>
                   <artifact>*:*</artifact>
                   <excludes>
+                    <exclude>assets/org/apache/commons/math3/exception/util/LocalizedFormats_fr.properties</exclude>
+                    <exclude>about.html</exclude>
+                    <exclude>mozilla/public-suffix-list.txt</exclude>
+
                     <exclude>META-INF/LICENSE*</exclude>
                     <exclude>META-INF/NOTICE*</exclude>
                     <exclude>META-INF/DEPENDENCIES</exclude>
@@ -775,6 +906,7 @@
                     <exclude>META-INF/*.SF</exclude>
                     <exclude>META-INF/*.DSA</exclude>
                     <exclude>META-INF/*.RSA</exclude>
+                    <exclude>google/protobuf/**/*.proto</exclude>
                   </excludes>
                 </filter>
                 <filter>
diff --git a/scripts/check_content.sh b/scripts/check_content.sh
index 09022d9d8..e4d3e2076 100755
--- a/scripts/check_content.sh
+++ b/scripts/check_content.sh
@@ -1,33 +1,40 @@
 #!/bin/bash -e
 
 # scripts used to check if all dependency is shaded into snowflake internal path
+# Do not add any additional exceptions into this script.
+# If this script fails the maven build, re-configure shading relocations in pom.xml.
 
 set -o pipefail
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-if jar tvf $DIR/../target/snowflake-ingest-sdk.jar  | awk '{print $8}' | grep -v -E "^(net|com)/snowflake" \
-    | grep -v -E "(com|net)/\$" | grep -v -E "^META-INF" | grep -v -E "^mozilla" | grep -v mime.types \
-    | grep -v project.properties | grep -v -E "javax" | grep -v -E "^org/" | grep -v -E "^com/google" \
-    | grep -v -E "^com/sun" | grep -v "log4j.properties" | grep -v "git.properties" | grep -v "io/" \
-    | grep -v "codegen/" | grep -v "com/codahale/" | grep -v "com/ibm/" | grep -v "LICENSE" | grep -v "aix/" \
-    | grep -v "darwin/" | grep -v "win/" | grep -v "freebsd/" | grep -v "linux/" | grep -v "com/github/" \
-    | grep -v -E "shaded/" | grep -v "webapps/"  | grep -v "microsoft/" | grep -v "com/ctc/" | grep -v "jersey/" \
-    | grep -v "edu/" | grep -v "com/jcraft/" | grep -v "contribs/" | grep -v "com/zaxxer/"  | grep -v "com/squareup/" \
-    | grep -v "com/thoughtworks/" | grep -v "com/jamesmurty/"  | grep -v "net/iharder/" \
-    | grep -v -E "^core-default.xml" \
-    | grep -v "yarn-version-info.properties" | grep -v "yarn-version-info.properties" \
-    | grep -v "about.html" | grep -v "jetty-dir.css" \
-    | grep -v "krb5_udp-template.conf" | grep -v "krb5-template.conf" \
-    | grep -v "ccache.txt" | grep -v "keytab.txt" \
-    | grep -v "common-version-info.properties" | grep -v "LocalizedFormats_fr.properties" \
-    | grep -v "org.apache.hadoop.application-classloader.properties" | grep -v "assets/" | grep -v "ehcache-core.xsd" \
-    | grep -v "ehcache-107ext.xsd" | grep -v "parquet.thrift" | grep -v "mapred-default.xml" \
-    | grep -v "yarn-default.xml" | grep -v ".keep"  | grep -v "NOTICE" | grep -v "digesterRules.xml" \
-    | grep -v "properties.dtd" | grep -v "PropertyList-1.0.dtd"  \
-    | grep -v "about.html" | grep -v "jetty-dir.css" | grep -v "krb5-template.conf" \
-    | grep -v "krb5_udp-template.conf" | grep -v "ccache.txt" | grep -v "keytab.txt" \
-    | grep -v "lookup.class" | grep -v "update.class" | grep -v "dig.class" | grep -v "jnamed"  \
-    ; then
-  echo "[ERROR] Ingest SDK jar includes class not under the snowflake namespace"
+
+if jar tvf $DIR/../target/snowflake-ingest-sdk.jar  | awk '{print $8}' | \
+    # Ignore directories
+    grep -v -E '/$' | \
+
+    # Snowflake top-level packages are allowed
+    grep -v -E "^net/snowflake/ingest" | \
+
+    # META-INF is allowed in the shaded JAR
+    grep -v -E "^META-INF" | \
+
+    # Files in the JAR root that are probably required
+    grep -v mime.types | \
+    grep -v project.properties | \
+    grep -v arrow-git.properties | \
+    grep -v core-default.xml | \
+    grep -v org.apache.hadoop.application-classloader.properties | \
+    grep -v common-version-info.properties | \
+    grep -v PropertyList-1.0.dtd | \
+    grep -v properties.dtd | \
+    grep -v parquet.thrift | \
+
+    # Native zstd libraries are allowed
+    grep -v -E '^darwin' | \
+    grep -v -E '^freebsd' | \
+    grep -v -E '^linux' | \
+    grep -v -E '^win' ; then
+
+  echo "[ERROR] JDBC jar includes class not under the snowflake namespace"
   exit 1
 fi
\ No newline at end of file
diff --git a/scripts/run_gh_actions.sh b/scripts/run_gh_actions.sh
index b7a0b632a..452d67682 100755
--- a/scripts/run_gh_actions.sh
+++ b/scripts/run_gh_actions.sh
@@ -1,3 +1,8 @@
+#!/bin/bash -e
+
+set -o pipefail
+
+# Build and install shaded JAR first. check_content.sh runs here.
 mvn install -DskipTests=true --batch-mode --show-version
 
 PARAMS=()

From 4b0709c45f87c8d8d1325fd40969484b9a4c1ea3 Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Mon, 8 May 2023 18:24:22 +0200
Subject: [PATCH 207/356] SNOW-806047 fix logged uncompressedSize for Parquet
 (#489)

---
 .../ingest/streaming/internal/ArrowFlusher.java          | 3 +++
 .../snowflake/ingest/streaming/internal/BlobBuilder.java | 2 +-
 .../ingest/streaming/internal/ChunkMetadata.java         | 2 +-
 .../net/snowflake/ingest/streaming/internal/Flusher.java | 3 +++
 .../ingest/streaming/internal/ParquetFlusher.java        | 9 ++++++++-
 5 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
index 42452cb55..061405a53 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
@@ -34,6 +34,7 @@ public Flusher.SerializationResult serialize(
     ByteArrayOutputStream chunkData = new ByteArrayOutputStream();
     List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
     long rowCount = 0L;
+    float chunkUncompressedSize = 0f;
     VectorSchemaRoot root = null;
     ArrowWriter arrowWriter = null;
     VectorLoader loader = null;
@@ -94,6 +95,7 @@ public Flusher.SerializationResult serialize(
         // Write channel data using the stream writer
         arrowWriter.writeBatch();
         rowCount += data.getRowCount();
+        chunkUncompressedSize += data.getBufferSize();
 
         logger.logDebug(
             "Finish building channel={}, rowCount={}, bufferSize={} in blob={}",
@@ -112,6 +114,7 @@ public Flusher.SerializationResult serialize(
         channelsMetadataList,
         columnEpStatsMapCombined,
         rowCount,
+        chunkUncompressedSize,
         chunkData,
         chunkMinMaxInsertTimeInMs);
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
index e1ce6b8bd..7d7c139c1 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
@@ -147,7 +147,7 @@ static <T> Blob constructBlobAndMetadata(
             firstChannelFlushContext.getFullyQualifiedTableName(),
             serializedChunk.rowCount,
             startOffset,
-            chunkData.size(),
+            serializedChunk.chunkUncompressedSize,
             compressedChunkLength,
             encryptedCompressedChunkDataSize,
             bdecVersion);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChunkMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChunkMetadata.java
index 582c7e272..ca2faaa1d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChunkMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChunkMetadata.java
@@ -32,7 +32,7 @@ static class Builder {
     private String schemaName;
     private String tableName;
     private Long chunkStartOffset;
-    private Integer chunkLength;
+    private Integer chunkLength; // compressedChunkLength
     private List<ChannelMetadata> channels;
     private String chunkMD5;
     private EpInfo epInfo;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
index 9923d2485..fc9d1c858 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
@@ -34,6 +34,7 @@ class SerializationResult {
     final List<ChannelMetadata> channelsMetadataList;
     final Map<String, RowBufferStats> columnEpStatsMapCombined;
     final long rowCount;
+    final float chunkUncompressedSize;
     final ByteArrayOutputStream chunkData;
     final Pair<Long, Long> chunkMinMaxInsertTimeInMs;
 
@@ -41,11 +42,13 @@ public SerializationResult(
         List<ChannelMetadata> channelsMetadataList,
         Map<String, RowBufferStats> columnEpStatsMapCombined,
         long rowCount,
+        float chunkUncompressedSize,
         ByteArrayOutputStream chunkData,
         Pair<Long, Long> chunkMinMaxInsertTimeInMs) {
       this.channelsMetadataList = channelsMetadataList;
       this.columnEpStatsMapCombined = columnEpStatsMapCombined;
       this.rowCount = rowCount;
+      this.chunkUncompressedSize = chunkUncompressedSize;
       this.chunkData = chunkData;
       this.chunkMinMaxInsertTimeInMs = chunkMinMaxInsertTimeInMs;
     }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
index 9382a673c..0ac28326d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
@@ -50,6 +50,7 @@ private SerializationResult serializeFromParquetWriteBuffers(
       throws IOException {
     List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
     long rowCount = 0L;
+    float chunkUncompressedSize = 0f;
     String firstChannelFullyQualifiedTableName = null;
     Map<String, RowBufferStats> columnEpStatsMapCombined = null;
     BdecParquetWriter mergedChannelWriter = null;
@@ -100,6 +101,7 @@ private SerializationResult serializeFromParquetWriteBuffers(
       }
 
       rowCount += data.getRowCount();
+      chunkUncompressedSize += data.getBufferSize();
 
       logger.logDebug(
           "Parquet Flusher: Finish building channel={}, rowCount={}, bufferSize={} in blob={}",
@@ -116,6 +118,7 @@ private SerializationResult serializeFromParquetWriteBuffers(
         channelsMetadataList,
         columnEpStatsMapCombined,
         rowCount,
+        chunkUncompressedSize,
         mergedChunkData,
         chunkMinMaxInsertTimeInMs);
   }
@@ -125,6 +128,7 @@ private SerializationResult serializeFromJavaObjects(
       throws IOException {
     List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
     long rowCount = 0L;
+    float chunkUncompressedSize = 0f;
     String firstChannelFullyQualifiedTableName = null;
     Map<String, RowBufferStats> columnEpStatsMapCombined = null;
     List<List<Object>> rows = null;
@@ -176,6 +180,7 @@ private SerializationResult serializeFromJavaObjects(
       rows.addAll(data.getVectors().rows);
 
       rowCount += data.getRowCount();
+      chunkUncompressedSize += data.getBufferSize();
 
       logger.logDebug(
           "Parquet Flusher: Finish building channel={}, rowCount={}, bufferSize={} in blob={},"
@@ -183,7 +188,8 @@ private SerializationResult serializeFromJavaObjects(
           data.getChannelContext().getFullyQualifiedName(),
           data.getRowCount(),
           data.getBufferSize(),
-          filePath);
+          filePath,
+          enableParquetInternalBuffering);
     }
 
     Map<String, String> metadata = channelsDataPerTable.get(0).getVectors().metadata;
@@ -196,6 +202,7 @@ private SerializationResult serializeFromJavaObjects(
         channelsMetadataList,
         columnEpStatsMapCombined,
         rowCount,
+        chunkUncompressedSize,
         mergedData,
         chunkMinMaxInsertTimeInMs);
   }

From 974bc315bbb7ab2c9b5c71eed87496aa2e8a5592 Mon Sep 17 00:00:00 2001
From: Waleed Fateem <72769898+sfc-gh-wfateem@users.noreply.github.com>
Date: Wed, 10 May 2023 11:39:06 -0500
Subject: [PATCH 208/356] SNOW-811371 Shade and Relocate Snowflake JDBC Classes
 (#494)

Users may run more than one application in a single JVM for
different purposes. If one application is built with the ingest
SDK and another one with the JDBC driver, then class conflicts
will occur.
---
 pom.xml | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/pom.xml b/pom.xml
index 196375b3b..afae7051c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -788,16 +788,19 @@
             <artifactId>maven-shade-plugin</artifactId>
             <version>3.4.1</version>
             <configuration>
-              <artifactSet>
-                <excludes>
-                  <exclude>net.snowflake:snowflake-jdbc</exclude>
-                </excludes>
-              </artifactSet>
               <relocations>
                 <relocation>
                   <pattern>com.nimbusds</pattern>
                   <shadedPattern>${shadeBase}.com.nimbusds</shadedPattern>
                 </relocation>
+                <relocation>
+                  <pattern>net.snowflake.client</pattern>
+                  <shadedPattern>${shadeBase}.net.snowflake.client</shadedPattern>
+                </relocation>
+                <relocation>
+                  <pattern>com.snowflake.client</pattern>
+                  <shadedPattern>${shadeBase}.com.snowflake.client</shadedPattern>
+                </relocation>
                 <relocation>
                   <pattern>net.jcip</pattern>
                   <shadedPattern>${shadeBase}.net.jcip</shadedPattern>

From 718614bac6138fbacab6e593f924da58ce73e574 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Fri, 21 Apr 2023 17:28:25 -0700
Subject: [PATCH 209/356] works now

---
 .../ingest/streaming/internal/BlobStats.java  | 36 ++++++++++---------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobStats.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobStats.java
index ccd2f9eeb..3628d0b13 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobStats.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobStats.java
@@ -5,19 +5,30 @@
 package net.snowflake.ingest.streaming.internal;
 
 import com.codahale.metrics.Timer;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import java.util.concurrent.TimeUnit;
 
 /** Latency information for a blob */
 class BlobStats {
-  private long buildDurationMs;
-  private long uploadDurationMs;
-
   // flush and register duration cannot be calculated in the client sdk we pass the start time
   // because the end time is when the request hits the server
   private long flushStartMs;
   private long registerStartMs;
 
+  private long buildDurationMs;
+  private long uploadDurationMs;
+
+  @JsonProperty("flush_start_ms")
+  long getFlushStartMs() {
+    return this.flushStartMs;
+  }
+
+  @JsonProperty("register_start_ms")
+  long getRegisterStartMs() {
+    return this.registerStartMs;
+  }
+
   @JsonProperty("build_duration_ms")
   long getBuildDurationMs() {
     return this.buildDurationMs;
@@ -28,14 +39,12 @@ long getUploadDurationMs() {
     return this.uploadDurationMs;
   }
 
-  @JsonProperty("flush_start_ms")
-  long getFlushStartMs() {
-    return this.flushStartMs;
+  void setFlushStartMs(long flushStartMs) {
+    this.flushStartMs = flushStartMs;
   }
 
-  @JsonProperty("register_start_ms")
-  long getRegisterStartMs() {
-    return this.registerStartMs;
+  void setRegisterStartMs(long registerStartMs) {
+    this.registerStartMs = registerStartMs;
   }
 
   void setBuildDurationMs(Timer.Context buildLatencyContext) {
@@ -44,17 +53,10 @@ void setBuildDurationMs(Timer.Context buildLatencyContext) {
     }
   }
 
+  @JsonIgnore
   void setUploadDurationMs(Timer.Context uploadLatencyContext) {
     if (uploadLatencyContext != null) {
       this.uploadDurationMs = TimeUnit.NANOSECONDS.toMillis(uploadLatencyContext.stop());
     }
   }
-
-  void setFlushStartMs(long flushStartMs) {
-    this.flushStartMs = flushStartMs;
-  }
-
-  void setRegisterStartMs(long registerStartMs) {
-    this.registerStartMs = registerStartMs;
-  }
 }

From 298dff7c97b7930606bab42b87b5539d3f12f79c Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Tue, 2 May 2023 16:54:54 -0700
Subject: [PATCH 210/356] remove register

---
 .../ingest/streaming/internal/BlobStats.java  | 47 ++++++++-----------
 .../streaming/internal/RegisterService.java   |  4 --
 2 files changed, 19 insertions(+), 32 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobStats.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobStats.java
index 3628d0b13..ba4f565e4 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobStats.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobStats.java
@@ -9,54 +9,45 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import java.util.concurrent.TimeUnit;
 
-/** Latency information for a blob */
+/** Latency stats for a blob */
 class BlobStats {
-  // flush and register duration cannot be calculated in the client sdk we pass the start time
-  // because the end time is when the request hits the server
+  // flush duration cannot be calculated in the client sdk, so we pass just the start time
   private long flushStartMs;
-  private long registerStartMs;
 
   private long buildDurationMs;
   private long uploadDurationMs;
 
-  @JsonProperty("flush_start_ms")
-  long getFlushStartMs() {
-    return this.flushStartMs;
-  }
-
-  @JsonProperty("register_start_ms")
-  long getRegisterStartMs() {
-    return this.registerStartMs;
-  }
-
   @JsonProperty("build_duration_ms")
   long getBuildDurationMs() {
     return this.buildDurationMs;
   }
 
-  @JsonProperty("upload_duration_ms")
-  long getUploadDurationMs() {
-    return this.uploadDurationMs;
-  }
-
-  void setFlushStartMs(long flushStartMs) {
-    this.flushStartMs = flushStartMs;
-  }
-
-  void setRegisterStartMs(long registerStartMs) {
-    this.registerStartMs = registerStartMs;
-  }
-
+  @JsonProperty("build_duration_ms")
   void setBuildDurationMs(Timer.Context buildLatencyContext) {
     if (buildLatencyContext != null) {
       this.buildDurationMs = TimeUnit.NANOSECONDS.toMillis(buildLatencyContext.stop());
     }
   }
 
-  @JsonIgnore
+  @JsonProperty("upload_duration_ms")
+  long getUploadDurationMs() {
+    return this.uploadDurationMs;
+  }
+
+  @JsonProperty("upload_duration_ms")
   void setUploadDurationMs(Timer.Context uploadLatencyContext) {
     if (uploadLatencyContext != null) {
       this.uploadDurationMs = TimeUnit.NANOSECONDS.toMillis(uploadLatencyContext.stop());
     }
   }
+
+  @JsonProperty("flush_start_ms")
+  long getFlushStartMs() {
+    return this.flushStartMs;
+  }
+
+  @JsonProperty("flush_start_ms")
+  void setFlushStartMs(long flushStartMs) {
+    this.flushStartMs = flushStartMs;
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index a48dbc469..44b90987b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -196,10 +196,6 @@ List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyT
             Timer.Context registerContext =
                 Utils.createTimerContext(this.owningClient.registerLatency);
 
-            for (BlobMetadata blobMetadata : blobs) {
-              blobMetadata.getBlobStats().setRegisterStartMs(System.currentTimeMillis());
-            }
-
             // Register the blobs, and invalidate any channels that return a failure status code
             this.owningClient.registerBlobs(blobs);
 

From 7b1d5fe3a89b8450972b227eebab4048de7e4d91 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Thu, 4 May 2023 13:19:05 -0700
Subject: [PATCH 211/356] clean up integration changes

---
 .../example/SnowflakeStreamingIngestExample.java     | 12 +++++-------
 .../ingest/streaming/internal/BlobStats.java         |  1 -
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
index d52655150..2b12aba9f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
+++ b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
@@ -24,13 +24,11 @@
  * <p>Please read the README.md file for detailed steps
  */
 public class SnowflakeStreamingIngestExample {
-  // TODO @rcheng - revert this before merge
   // Please follow the example in profile_streaming.json.example to see the required properties, or
   // if you have already set up profile.json with Snowpipe before, all you need is to add the "role"
   // property.
   private static String PROFILE_PATH = "profile.json";
   private static final ObjectMapper mapper = new ObjectMapper();
-  private static final String TABLE_NAME = "revi_ingest_1";
 
   public static void main(String[] args) throws Exception {
     Properties props = new Properties();
@@ -49,10 +47,10 @@ public static void main(String[] args) throws Exception {
       // db/schema/table needs to be present
       // Example: create or replace table MY_TABLE(c1 number);
       OpenChannelRequest request1 =
-          OpenChannelRequest.builder("revi_channel")
-              .setDBName(props.getProperty("database"))
-              .setSchemaName(props.getProperty("schema"))
-              .setTableName(TABLE_NAME)
+          OpenChannelRequest.builder("MY_CHANNEL")
+              .setDBName("MY_DATABASE")
+              .setSchemaName("MY_SCHEMA")
+              .setTableName("MY_TABLE")
               .setOnErrorOption(
                   OpenChannelRequest.OnErrorOption.CONTINUE) // Another ON_ERROR option is ABORT
               .build();
@@ -80,7 +78,7 @@ public static void main(String[] args) throws Exception {
       // If needed, you can check the offset_token registered in Snowflake to make sure everything
       // is committed
       final int expectedOffsetTokenInSnowflake = totalRowsInTable - 1; // 0 based offset_token
-      final int maxRetries = 5;
+      final int maxRetries = 10;
       int retryCount = 0;
 
       do {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobStats.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobStats.java
index ba4f565e4..42994bbb4 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobStats.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobStats.java
@@ -5,7 +5,6 @@
 package net.snowflake.ingest.streaming.internal;
 
 import com.codahale.metrics.Timer;
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import java.util.concurrent.TimeUnit;
 

From bcd8a32b0ec030aa40877f50a931ef3ef80c917b Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 17 May 2023 13:51:03 -0700
Subject: [PATCH 212/356] NO-SNOW: Update MAX_CHUNK_SIZE_IN_BYTES default to be
 32MB and make it configurable  (#498)

We have seen many cases where the BDEC file is too small and causing high cost, so increasing the max chunk size from 16MB to 32MB. To avoid regressions that prevent customers to upgrade, we make the value configurable.
---
 .../streaming/internal/AbstractRowBuffer.java |  6 ++++--
 .../streaming/internal/ParquetFlusher.java    |  8 +++++--
 .../streaming/internal/ParquetRowBuffer.java  | 21 ++++++++++++-------
 ...owflakeStreamingIngestChannelInternal.java |  9 +++++---
 .../net/snowflake/ingest/utils/Constants.java |  1 -
 .../ingest/utils/ParameterProvider.java       | 13 +++++++++++-
 .../parquet/hadoop/BdecParquetWriter.java     | 13 ++++++------
 .../internal/ParameterProviderTest.java       |  5 +++++
 .../streaming/internal/RowBufferTest.java     |  4 +++-
 9 files changed, 56 insertions(+), 24 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 9156f9c4b..0580006d0 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -559,7 +559,8 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelRuntimeState,
-      boolean enableParquetMemoryOptimization) {
+      boolean enableParquetMemoryOptimization,
+      long maxChunkSizeInBytes) {
     switch (bdecVersion) {
       case ONE:
         //noinspection unchecked
@@ -581,7 +582,8 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
                 fullyQualifiedChannelName,
                 rowSizeMetric,
                 channelRuntimeState,
-                enableParquetMemoryOptimization);
+                enableParquetMemoryOptimization,
+                maxChunkSizeInBytes);
       default:
         throw new SFException(
             ErrorCode.INTERNAL_ERROR, "Unsupported BDEC format version: " + bdecVersion);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
index 0ac28326d..b0c14b12f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
@@ -25,14 +25,17 @@ public class ParquetFlusher implements Flusher<ParquetChunkData> {
   private static final Logging logger = new Logging(ParquetFlusher.class);
   private final MessageType schema;
   private final boolean enableParquetInternalBuffering;
+  private final long maxChunkSizeInBytes;
 
   /**
    * Construct parquet flusher from its schema and set flag that indicates whether Parquet memory
    * optimization is enabled, i.e. rows will be buffered in internal Parquet buffer.
    */
-  public ParquetFlusher(MessageType schema, boolean enableParquetInternalBuffering) {
+  public ParquetFlusher(
+      MessageType schema, boolean enableParquetInternalBuffering, long maxChunkSizeInBytes) {
     this.schema = schema;
     this.enableParquetInternalBuffering = enableParquetInternalBuffering;
+    this.maxChunkSizeInBytes = maxChunkSizeInBytes;
   }
 
   @Override
@@ -194,7 +197,8 @@ private SerializationResult serializeFromJavaObjects(
 
     Map<String, String> metadata = channelsDataPerTable.get(0).getVectors().metadata;
     parquetWriter =
-        new BdecParquetWriter(mergedData, schema, metadata, firstChannelFullyQualifiedTableName);
+        new BdecParquetWriter(
+            mergedData, schema, metadata, firstChannelFullyQualifiedTableName, maxChunkSizeInBytes);
     rows.forEach(parquetWriter::writeRow);
     parquetWriter.close();
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index a45a52b93..73774d080 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -52,6 +52,8 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
 
   private MessageType schema;
   private final boolean enableParquetInternalBuffering;
+  private final long maxChunkSizeInBytes;
+
   /** Construct a ParquetRowBuffer object. */
   ParquetRowBuffer(
       OpenChannelRequest.OnErrorOption onErrorOption,
@@ -60,7 +62,8 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelRuntimeState,
-      boolean enableParquetInternalBuffering) {
+      boolean enableParquetInternalBuffering,
+      long maxChunkSizeInBytes) {
     super(
         onErrorOption,
         defaultTimezone,
@@ -68,12 +71,13 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
         fullyQualifiedChannelName,
         rowSizeMetric,
         channelRuntimeState);
-    fieldIndex = new HashMap<>();
-    metadata = new HashMap<>();
-    data = new ArrayList<>();
-    tempData = new ArrayList<>();
-    channelName = fullyQualifiedChannelName;
+    this.fieldIndex = new HashMap<>();
+    this.metadata = new HashMap<>();
+    this.data = new ArrayList<>();
+    this.tempData = new ArrayList<>();
+    this.channelName = fullyQualifiedChannelName;
     this.enableParquetInternalBuffering = enableParquetInternalBuffering;
+    this.maxChunkSizeInBytes = maxChunkSizeInBytes;
   }
 
   @Override
@@ -117,7 +121,8 @@ private void createFileWriter() {
     fileOutput = new ByteArrayOutputStream();
     try {
       if (enableParquetInternalBuffering) {
-        bdecParquetWriter = new BdecParquetWriter(fileOutput, schema, metadata, channelName);
+        bdecParquetWriter =
+            new BdecParquetWriter(fileOutput, schema, metadata, channelName, maxChunkSizeInBytes);
       } else {
         this.bdecParquetWriter = null;
       }
@@ -305,7 +310,7 @@ void closeInternal() {
 
   @Override
   public Flusher<ParquetChunkData> createFlusher() {
-    return new ParquetFlusher(schema, enableParquetInternalBuffering);
+    return new ParquetFlusher(schema, enableParquetInternalBuffering, maxChunkSizeInBytes);
   }
 
   private static class ParquetColumn {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 3916bc0d9..f4c4c6952 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -5,7 +5,6 @@
 package net.snowflake.ingest.streaming.internal;
 
 import static net.snowflake.ingest.utils.Constants.INSERT_THROTTLE_MAX_RETRY_COUNT;
-import static net.snowflake.ingest.utils.Constants.MAX_CHUNK_SIZE_IN_BYTES;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
 import static net.snowflake.ingest.utils.ParameterProvider.MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT;
 
@@ -131,7 +130,10 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
             channelState,
             owningClient != null
                 ? owningClient.getParameterProvider().getEnableParquetInternalBuffering()
-                : ParameterProvider.ENABLE_PARQUET_INTERNAL_BUFFERING_DEFAULT);
+                : ParameterProvider.ENABLE_PARQUET_INTERNAL_BUFFERING_DEFAULT,
+            owningClient != null
+                ? owningClient.getParameterProvider().getMaxChunkSizeInBytes()
+                : ParameterProvider.MAX_CHUNK_SIZE_IN_BYTES_DEFAULT);
     logger.logInfo(
         "Channel={} created for table={}",
         this.channelFlushContext.getName(),
@@ -365,7 +367,8 @@ public InsertValidationResponse insertRows(
     // Start flush task if the chunk size reaches a certain size
     // TODO: Checking table/chunk level size reduces throughput a lot, we may want to check it only
     // if a large number of rows are inserted
-    if (this.rowBuffer.getSize() >= MAX_CHUNK_SIZE_IN_BYTES) {
+    if (this.rowBuffer.getSize()
+        >= this.owningClient.getParameterProvider().getMaxChunkSizeInBytes()) {
       this.owningClient.setNeedFlush();
     }
 
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index e4e399978..050f14264 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -34,7 +34,6 @@ public class Constants {
   public static final int BLOB_UPLOAD_TIMEOUT_IN_SEC = 5;
   public static final int INSERT_THROTTLE_MAX_RETRY_COUNT = 60;
   public static final long MAX_BLOB_SIZE_IN_BYTES = 256000000L;
-  public static final long MAX_CHUNK_SIZE_IN_BYTES = 16000000L;
   public static final int BLOB_TAG_SIZE_IN_BYTES = 4;
   public static final int BLOB_VERSION_SIZE_IN_BYTES = 1;
   public static final int BLOB_FILE_SIZE_SIZE_IN_BYTES = 8;
diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
index 11f9fbeac..e2f38f476 100644
--- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
+++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -16,7 +16,6 @@ public class ParameterProvider {
       "STREAMING_INGEST_CLIENT_SDK_INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE".toLowerCase();
   public static final String INSERT_THROTTLE_THRESHOLD_IN_BYTES =
       "STREAMING_INGEST_CLIENT_SDK_INSERT_THROTTLE_THRESHOLD_IN_BYTES".toLowerCase();
-
   public static final String ENABLE_SNOWPIPE_STREAMING_METRICS =
       "ENABLE_SNOWPIPE_STREAMING_JMX_METRICS".toLowerCase();
   public static final String BLOB_FORMAT_VERSION = "BLOB_FORMAT_VERSION".toLowerCase();
@@ -26,6 +25,7 @@ public class ParameterProvider {
   public static final String MAX_MEMORY_LIMIT_IN_BYTES = "MAX_MEMORY_LIMIT_IN_BYTES".toLowerCase();
   public static final String ENABLE_PARQUET_INTERNAL_BUFFERING =
       "ENABLE_PARQUET_INTERNAL_BUFFERING".toLowerCase();
+  public static final String MAX_CHUNK_SIZE_IN_BYTES = "MAX_CHUNK_SIZE_IN_BYTES".toLowerCase();
 
   // Default values
   public static final long BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT = 1000;
@@ -39,6 +39,7 @@ public class ParameterProvider {
   public static final int IO_TIME_CPU_RATIO_DEFAULT = 2;
   public static final int BLOB_UPLOAD_MAX_RETRY_COUNT_DEFAULT = 24;
   public static final long MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT = -1L;
+  public static final long MAX_CHUNK_SIZE_IN_BYTES_DEFAULT = 32000000L;
 
   /* Parameter that enables using internal Parquet buffers for buffering of rows before serializing.
   It reduces memory consumption compared to using Java Objects for buffering.*/
@@ -136,6 +137,9 @@ private void setParameterMap(Map<String, Object> parameterOverrides, Properties
         ENABLE_PARQUET_INTERNAL_BUFFERING_DEFAULT,
         parameterOverrides,
         props);
+
+    this.updateValue(
+        MAX_CHUNK_SIZE_IN_BYTES, MAX_CHUNK_SIZE_IN_BYTES_DEFAULT, parameterOverrides, props);
   }
 
   /** @return Longest interval in milliseconds between buffer flushes */
@@ -261,6 +265,13 @@ public boolean getEnableParquetInternalBuffering() {
     return (val instanceof String) ? Boolean.parseBoolean(val.toString()) : (boolean) val;
   }
 
+  /** @return The max chunk size in bytes */
+  public long getMaxChunkSizeInBytes() {
+    Object val =
+        this.parameterMap.getOrDefault(MAX_CHUNK_SIZE_IN_BYTES, MAX_CHUNK_SIZE_IN_BYTES_DEFAULT);
+    return (val instanceof String) ? Long.parseLong(val.toString()) : (long) val;
+  }
+
   @Override
   public String toString() {
     return "ParameterProvider{" + "parameterMap=" + parameterMap + '}';
diff --git a/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java b/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java
index 9122e20be..55ab1bf30 100644
--- a/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java
+++ b/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java
@@ -4,8 +4,6 @@
 
 package org.apache.parquet.hadoop;
 
-import static net.snowflake.ingest.utils.Constants.MAX_CHUNK_SIZE_IN_BYTES;
-
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.List;
@@ -52,9 +50,10 @@ public BdecParquetWriter(
       ByteArrayOutputStream stream,
       MessageType schema,
       Map<String, String> extraMetaData,
-      String channelName)
+      String channelName,
+      long maxChunkSizeInBytes)
       throws IOException {
-    OutputFile file = new ByteArrayOutputFile(stream);
+    OutputFile file = new ByteArrayOutputFile(stream, maxChunkSizeInBytes);
     ParquetProperties encodingProps = createParquetProperties();
     Configuration conf = new Configuration();
     WriteSupport<List<Object>> writeSupport =
@@ -166,9 +165,11 @@ private static ParquetProperties createParquetProperties() {
    */
   private static class ByteArrayOutputFile implements OutputFile {
     private final ByteArrayOutputStream stream;
+    private final long maxChunkSizeInBytes;
 
-    private ByteArrayOutputFile(ByteArrayOutputStream stream) {
+    private ByteArrayOutputFile(ByteArrayOutputStream stream, long maxChunkSizeInBytes) {
       this.stream = stream;
+      this.maxChunkSizeInBytes = maxChunkSizeInBytes;
     }
 
     @Override
@@ -189,7 +190,7 @@ public boolean supportsBlockSize() {
 
     @Override
     public long defaultBlockSize() {
-      return (int) MAX_CHUNK_SIZE_IN_BYTES;
+      return maxChunkSizeInBytes;
     }
   }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
index f7f8da84f..1fe034635 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
@@ -21,6 +21,7 @@ public void withValuesSet() {
     parameterMap.put(ParameterProvider.IO_TIME_CPU_RATIO, 10);
     parameterMap.put(ParameterProvider.BLOB_UPLOAD_MAX_RETRY_COUNT, 100);
     parameterMap.put(ParameterProvider.MAX_MEMORY_LIMIT_IN_BYTES, 1000L);
+    parameterMap.put(ParameterProvider.MAX_CHUNK_SIZE_IN_BYTES, 1000000L);
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
 
     Assert.assertEquals(3L, parameterProvider.getBufferFlushIntervalInMs());
@@ -31,6 +32,7 @@ public void withValuesSet() {
     Assert.assertEquals(10, parameterProvider.getIOTimeCpuRatio());
     Assert.assertEquals(100, parameterProvider.getBlobUploadMaxRetryCount());
     Assert.assertEquals(1000L, parameterProvider.getMaxMemoryLimitInBytes());
+    Assert.assertEquals(1000000L, parameterProvider.getMaxChunkSizeInBytes());
   }
 
   @Test
@@ -117,5 +119,8 @@ public void withDefaultValues() {
     Assert.assertEquals(
         ParameterProvider.MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT,
         parameterProvider.getMaxMemoryLimitInBytes());
+    Assert.assertEquals(
+        ParameterProvider.MAX_CHUNK_SIZE_IN_BYTES_DEFAULT,
+        parameterProvider.getMaxChunkSizeInBytes());
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 55df1eb21..b6143e6ce 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -1,6 +1,7 @@
 package net.snowflake.ingest.streaming.internal;
 
 import static java.time.ZoneOffset.UTC;
+import static net.snowflake.ingest.utils.ParameterProvider.MAX_CHUNK_SIZE_IN_BYTES_DEFAULT;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -131,7 +132,8 @@ private AbstractRowBuffer<?> createTestBuffer(OpenChannelRequest.OnErrorOption o
         "test.buffer",
         rs -> {},
         initialState,
-        enableParquetMemoryOptimization);
+        enableParquetMemoryOptimization,
+        MAX_CHUNK_SIZE_IN_BYTES_DEFAULT);
   }
 
   @Test

From fec721bf6128d436c008cb2794637c9c1769bec2 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 18 May 2023 17:51:58 -0700
Subject: [PATCH 213/356] SNOW-758492: avoid combining channels with different
 schema (#501)

We could have two channels with different schema when the older channel haven't got invalidated yet due to schema changes, avoid combining these two channels into the same chunk so the newer channel's data could be committed
---
 .../streaming/internal/FlushService.java      | 39 +++++---
 .../streaming/internal/FlushServiceTest.java  | 94 +++++++++++++++----
 2 files changed, 103 insertions(+), 30 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index f738d33be..7f38c1324 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -374,27 +374,23 @@ void distributeFlushTasks() {
           int idx = 0;
           while (idx < channelsDataPerTable.size()) {
             ChannelData<T> channelData = channelsDataPerTable.get(idx);
-            // Stop processing the rest of channels if reaching the blob size limit or the channel
-            // has different encryption key ids
+            // Stop processing the rest of channels when needed
             if (idx > 0
-                && (totalBufferSizeInBytes + channelData.getBufferSize() > MAX_BLOB_SIZE_IN_BYTES
-                    || !Objects.equals(
-                        channelData.getChannelContext().getEncryptionKeyId(),
-                        channelsDataPerTable
-                            .get(idx - 1)
-                            .getChannelContext()
-                            .getEncryptionKeyId()))) {
+                && shouldStopProcessing(
+                    totalBufferSizeInBytes, channelData, channelsDataPerTable.get(idx - 1))) {
               leftoverChannelsDataPerTable.addAll(
                   channelsDataPerTable.subList(idx, channelsDataPerTable.size()));
               logger.logInfo(
                   "Creation of another blob is needed because of blob size limit or different"
-                      + " encryption ids, client={}, table={},  size={}, encryptionId1={},"
-                      + " encryptionId2={}",
+                      + " encryption ids or different schema, client={}, table={},  size={},"
+                      + " encryptionId1={}, encryptionId2={}, schema1={}, schema2={}",
                   this.owningClient.getName(),
                   channelData.getChannelContext().getTableName(),
                   totalBufferSizeInBytes + channelData.getBufferSize(),
                   channelData.getChannelContext().getEncryptionKeyId(),
-                  channelsDataPerTable.get(idx - 1).getChannelContext().getEncryptionKeyId());
+                  channelsDataPerTable.get(idx - 1).getChannelContext().getEncryptionKeyId(),
+                  channelData.getColumnEps().keySet(),
+                  channelsDataPerTable.get(idx - 1).getColumnEps().keySet());
               break;
             }
             totalBufferSizeInBytes += channelData.getBufferSize();
@@ -475,6 +471,25 @@ void distributeFlushTasks() {
     this.registerService.addBlobs(blobs);
   }
 
+  /**
+   * Check whether we should stop merging more channels into the same chunk, we need to stop in a
+   * few cases:
+   *
+   * <p>When the size is larger than a certain threshold
+   *
+   * <p>When the encryption key ids are not the same
+   *
+   * <p>When the schemas are not the same
+   */
+  private boolean shouldStopProcessing(
+      float totalBufferSizeInBytes, ChannelData<T> current, ChannelData<T> prev) {
+    return totalBufferSizeInBytes + current.getBufferSize() > MAX_BLOB_SIZE_IN_BYTES
+        || !Objects.equals(
+            current.getChannelContext().getEncryptionKeyId(),
+            prev.getChannelContext().getEncryptionKeyId())
+        || !current.getColumnEps().keySet().equals(prev.getColumnEps().keySet());
+  }
+
   /**
    * Builds and uploads file to cloud storage.
    *
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 338d8665a..12e77ab7b 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -403,9 +403,9 @@ private SnowflakeStreamingIngestChannelInternal<?> addChannel4(TestContext<?> te
         .buildAndAdd();
   }
 
-  private static ColumnMetadata createTestIntegerColumn() {
+  private static ColumnMetadata createTestIntegerColumn(String name) {
     ColumnMetadata colInt = new ColumnMetadata();
-    colInt.setName("COLINT");
+    colInt.setName(name);
     colInt.setPhysicalType("SB4");
     colInt.setNullable(true);
     colInt.setLogicalType("FIXED");
@@ -414,9 +414,9 @@ private static ColumnMetadata createTestIntegerColumn() {
     return colInt;
   }
 
-  private static ColumnMetadata createTestTextColumn() {
+  private static ColumnMetadata createTestTextColumn(String name) {
     ColumnMetadata colChar = new ColumnMetadata();
-    colChar.setName("COLCHAR");
+    colChar.setName(name);
     colChar.setPhysicalType("LOB");
     colChar.setNullable(true);
     colChar.setLogicalType("TEXT");
@@ -493,19 +493,22 @@ public void testBlobCreation() throws Exception {
     SnowflakeStreamingIngestChannelInternal<?> channel1 = addChannel1(testContext);
     SnowflakeStreamingIngestChannelInternal<?> channel2 = addChannel2(testContext);
     SnowflakeStreamingIngestChannelInternal<?> channel4 = addChannel4(testContext);
+    String colName1 = "testBlobCreation1";
+    String colName2 = "testBlobCreation2";
 
-    List<ColumnMetadata> schema = Arrays.asList(createTestIntegerColumn(), createTestTextColumn());
+    List<ColumnMetadata> schema =
+        Arrays.asList(createTestIntegerColumn(colName1), createTestTextColumn(colName2));
     channel1.getRowBuffer().setupSchema(schema);
     channel2.getRowBuffer().setupSchema(schema);
     channel4.getRowBuffer().setupSchema(schema);
 
     List<Map<String, Object>> rows1 =
         RowSetBuilder.newBuilder()
-            .addColumn("COLINT", 11)
-            .addColumn("COLCHAR", "bob")
+            .addColumn(colName1, 11)
+            .addColumn(colName2, "bob")
             .newRow()
-            .addColumn("COLINT", 22)
-            .addColumn("COLCHAR", "bob")
+            .addColumn(colName1, 22)
+            .addColumn(colName2, "bob")
             .build();
 
     channel1.insertRows(rows1, "offset1");
@@ -519,6 +522,55 @@ public void testBlobCreation() throws Exception {
     Mockito.verify(flushService, Mockito.atLeast(2)).buildAndUpload(Mockito.any(), Mockito.any());
   }
 
+  @Test
+  public void testBlobSplitDueToDifferentSchema() throws Exception {
+    TestContext<?> testContext = testContextFactory.create();
+    SnowflakeStreamingIngestChannelInternal<?> channel1 = addChannel1(testContext);
+    SnowflakeStreamingIngestChannelInternal<?> channel2 = addChannel2(testContext);
+    String colName1 = "testBlobSplitDueToDifferentSchema1";
+    String colName2 = "testBlobSplitDueToDifferentSchema2";
+    String colName3 = "testBlobSplitDueToDifferentSchema3";
+
+    List<ColumnMetadata> schema1 =
+        Arrays.asList(createTestIntegerColumn(colName1), createTestTextColumn(colName2));
+    List<ColumnMetadata> schema2 =
+        Arrays.asList(
+            createTestIntegerColumn(colName1),
+            createTestTextColumn(colName2),
+            createTestIntegerColumn(colName3));
+    channel1.getRowBuffer().setupSchema(schema1);
+    channel2.getRowBuffer().setupSchema(schema2);
+
+    List<Map<String, Object>> rows1 =
+        RowSetBuilder.newBuilder()
+            .addColumn(colName1, 11)
+            .addColumn(colName2, "bob")
+            .newRow()
+            .addColumn(colName1, 22)
+            .addColumn(colName2, "bob")
+            .build();
+
+    List<Map<String, Object>> rows2 =
+        RowSetBuilder.newBuilder()
+            .addColumn(colName1, 11)
+            .addColumn(colName2, "bob")
+            .addColumn(colName3, 11)
+            .newRow()
+            .addColumn(colName1, 22)
+            .addColumn(colName2, "bob")
+            .addColumn(colName3, 22)
+            .build();
+
+    channel1.insertRows(rows1, "offset1");
+    channel2.insertRows(rows2, "offset2");
+
+    FlushService<?> flushService = testContext.flushService;
+
+    // Force = true flushes
+    flushService.flush(true).get();
+    Mockito.verify(flushService, Mockito.atLeast(2)).buildAndUpload(Mockito.any(), Mockito.any());
+  }
+
   @Test
   public void testBuildAndUpload() throws Exception {
     long expectedBuildLatencyMs = 100;
@@ -527,21 +579,24 @@ public void testBuildAndUpload() throws Exception {
     TestContext<?> testContext = testContextFactory.create();
     SnowflakeStreamingIngestChannelInternal<?> channel1 = addChannel1(testContext);
     SnowflakeStreamingIngestChannelInternal<?> channel2 = addChannel2(testContext);
+    String colName1 = "testBuildAndUpload1";
+    String colName2 = "testBuildAndUpload2";
 
-    List<ColumnMetadata> schema = Arrays.asList(createTestIntegerColumn(), createTestTextColumn());
+    List<ColumnMetadata> schema =
+        Arrays.asList(createTestIntegerColumn(colName1), createTestTextColumn(colName2));
     channel1.getRowBuffer().setupSchema(schema);
     channel2.getRowBuffer().setupSchema(schema);
 
     List<Map<String, Object>> rows1 =
         RowSetBuilder.newBuilder()
-            .addColumn("COLINT", 11)
-            .addColumn("COLCHAR", "bob")
+            .addColumn(colName1, 11)
+            .addColumn(colName2, "bob")
             .newRow()
-            .addColumn("COLINT", 22)
-            .addColumn("COLCHAR", "bob")
+            .addColumn(colName1, 22)
+            .addColumn(colName2, "bob")
             .build();
     List<Map<String, Object>> rows2 =
-        RowSetBuilder.newBuilder().addColumn("COLINT", null).addColumn("COLCHAR", "toby").build();
+        RowSetBuilder.newBuilder().addColumn(colName1, null).addColumn(colName2, "toby").build();
 
     channel1.insertRows(rows1, "offset1");
     channel2.insertRows(rows2, "offset2");
@@ -672,15 +727,18 @@ public void testBuildErrors() throws Exception {
     TestContext<?> testContext = testContextFactory.create();
     SnowflakeStreamingIngestChannelInternal<?> channel1 = addChannel1(testContext);
     SnowflakeStreamingIngestChannelInternal<?> channel3 = addChannel3(testContext);
+    String colName1 = "testBuildErrors1";
+    String colName2 = "testBuildErrors2";
 
-    List<ColumnMetadata> schema = Arrays.asList(createTestIntegerColumn(), createTestTextColumn());
+    List<ColumnMetadata> schema =
+        Arrays.asList(createTestIntegerColumn(colName1), createTestTextColumn(colName2));
     channel1.getRowBuffer().setupSchema(schema);
     channel3.getRowBuffer().setupSchema(schema);
 
     List<Map<String, Object>> rows1 =
-        RowSetBuilder.newBuilder().addColumn("COLINT", 0).addColumn("COLCHAR", "alice").build();
+        RowSetBuilder.newBuilder().addColumn(colName1, 0).addColumn(colName2, "alice").build();
     List<Map<String, Object>> rows2 =
-        RowSetBuilder.newBuilder().addColumn("COLINT", 0).addColumn("COLCHAR", 111).build();
+        RowSetBuilder.newBuilder().addColumn(colName1, 0).addColumn(colName2, 111).build();
 
     channel1.insertRows(rows1, "offset1");
     channel3.insertRows(rows2, "offset2");

From c3f1fd653cbf0b1321a2d2469d550611f645633a Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Fri, 12 May 2023 19:07:21 +0000
Subject: [PATCH 214/356] added snapshot_deploy.sh file and added snapshot repo
 id to pom file

---
 pom.xml            | 35 +++++++++++++++++++
 snapshot_deploy.sh | 85 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 120 insertions(+)
 create mode 100755 snapshot_deploy.sh

diff --git a/pom.xml b/pom.xml
index afae7051c..fafdd3569 100644
--- a/pom.xml
+++ b/pom.xml
@@ -977,6 +977,41 @@
         </plugins>
       </build>
     </profile>
+    <profile>
+      <id>snapshot-deploy</id>
+      <activation>
+        <property>
+          <name>snapshot-deploy</name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-gpg-plugin</artifactId>
+            <version>1.6</version>
+            <executions>
+              <execution>
+                <goals>
+                  <goal>sign-and-deploy-file</goal>
+                </goals>
+                <phase>deploy</phase>
+                <configuration>
+                  <file>target/${project.artifactId}.jar</file>
+                  <repositoryId>snapshot</repositoryId>
+                  <url>https://nexus.int.snowflakecomputing.com/repository/Snapshots/</url>
+                  <pomFile>generated_public_pom.xml</pomFile>
+                  <javadoc>target/${project.artifactId}-javadoc.jar</javadoc>
+                  <sources>target/${project.artifactId}-sources.jar</sources>
+                  <keyname>${env.GPG_KEY_ID}</keyname>
+                  <passphrase>${env.GPG_KEY_PASSPHRASE}</passphrase>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <profile>
       <id>ghActionsIT</id>
       <activation>
diff --git a/snapshot_deploy.sh b/snapshot_deploy.sh
new file mode 100755
index 000000000..f0af3a9df
--- /dev/null
+++ b/snapshot_deploy.sh
@@ -0,0 +1,85 @@
+#!/bin/bash -e
+
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+export GPG_KEY_ID="Snowflake Computing"
+export SONATYPE_USER="$sonatype_user"
+export SONATYPE_PWD="$sonatype_password"
+
+if [ -z "$GPG_KEY_PASSPHRASE" ]; then
+  echo "[ERROR] GPG passphrase is not specified for $GPG_KEY_ID!"
+  exit 1
+fi
+
+if [ -z "$GPG_PRIVATE_KEY" ]; then
+  echo "[ERROR] GPG private key file is not specified!"
+  exit 1
+fi
+
+echo "[INFO] Import PGP Key"
+if ! gpg --list-secret-key | grep "$GPG_KEY_ID"; then
+  gpg --allow-secret-key-import --import "$GPG_PRIVATE_KEY"
+fi
+
+# copy the settings.xml template and inject credential information
+SNAPSHOT_DEPLOY_SETTINGS_XML="$THIS_DIR/mvn_settings_snapshot_deploy.xml"
+MVN_REPOSITORY_ID=snapshot
+
+cat > $SNAPSHOT_DEPLOY_SETTINGS_XML << SETTINGS.XML
+<?xml version="1.0" encoding="UTF-8"?>
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+     xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
+  <servers>
+    <server>
+      <id>$MVN_REPOSITORY_ID</id>
+      <username>$SONATYPE_USER</username>
+      <password>$SONATYPE_PWD</password>
+    </server>
+  </servers>
+</settings>
+SETTINGS.XML
+
+MVN_OPTIONS+=(
+  "--settings" "$SNAPSHOT_DEPLOY_SETTINGS_XML"
+  "--batch-mode"
+)
+
+echo "[Info] Sign package and deploy to staging area"
+project_version=$($THIS_DIR/scripts/get_project_info_from_pom.py $THIS_DIR/pom.xml version)
+$THIS_DIR/scripts/update_project_version.py public_pom.xml $project_version > generated_public_pom.xml
+
+mvn deploy ${MVN_OPTIONS[@]} -Dsnapshot-deploy
+
+echo "[INFO] Close and Release"
+snowflake_repositories=$(mvn ${MVN_OPTIONS[@]} \
+    org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-list \
+    -DserverId=$MVN_REPOSITORY_ID versions:set -DnewVersion=$project_version-SNAPSHOT
+    -DnexusUrl=https://nexus.int.snowflakecomputing.com/| grep netsnowflake | awk '{print $2}')
+IFS=" "
+if (( $(echo $snowflake_repositories | wc -l)!=1 )); then
+    echo "[ERROR] Not single netsnowflake repository is staged. Login https://nexus.int.snowflakecomputing.com/ and make sure no netsnowflake remains there."
+    exit 1
+fi
+if ! mvn ${MVN_OPTIONS[@]} \
+    org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-close \
+    -DserverId=$MVN_REPOSITORY_ID versions:set -DnewVersion=$project_version-SNAPSHOT\
+    -DnexusUrl=https://nexus.int.snowflakecomputing.com/ \
+    -DstagingRepositoryId=$snowflake_repositories \
+    -DstagingDescription="Automated Close"; then
+    echo "[ERROR] Failed to close. Fix the errors and try this script again"
+    mvn ${MVN_OPTIONS[@]} \
+        nexus-staging:rc-drop \
+        -DserverId=$MVN_REPOSITORY_ID versions:set -DnewVersion=$project_version-SNAPSHOT\
+        -DnexusUrl=https://nexus.int.snowflakecomputing.com/ \
+        -DstagingRepositoryId=$snowflake_repositories \
+        -DstagingDescription="Failed to close. Dropping..."
+fi
+
+mvn ${MVN_OPTIONS[@]} \
+    org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-release \
+    -DserverId=$MVN_REPOSITORY_ID versions:set -DnewVersion=$project_version-SNAPSHOT\
+    -DnexusUrl=https://nexus.int.snowflakecomputing.com/ \
+    -DstagingRepositoryId=$snowflake_repositories \
+    -DstagingDescription="Automated Release"
+
+rm $SNAPSHOT_DEPLOY_SETTINGS_XML
\ No newline at end of file

From 12ae3cd13f26500c19f40705afa5f1e07a201857 Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Fri, 12 May 2023 20:17:13 +0000
Subject: [PATCH 215/356] added maven surfire plugin dependency

---
 pom.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/pom.xml b/pom.xml
index fafdd3569..a823131b8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -511,6 +511,7 @@
             <skip>true</skip>
           </configuration>
         </plugin>
+
         <!-- Need to define the maven-failsafe-plugin here -->
         <!-- http://maven.apache.org/surefire/maven-failsafe-plugin/plugin-info.html -->
         <plugin>
@@ -518,6 +519,11 @@
           <artifactId>maven-failsafe-plugin</artifactId>
           <version>3.0.0-M5</version>
         </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-surefire-plugin</artifactId>
+          <version>2.19.1</version>
+        </plugin>
         <plugin>
           <groupId>org.jacoco</groupId>
           <artifactId>jacoco-maven-plugin</artifactId>

From b0c239ac888296c507ce3c201a20161f5bf61a44 Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Fri, 12 May 2023 20:31:00 +0000
Subject: [PATCH 216/356] modified snapshot_deploy.sh file

---
 snapshot_deploy.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/snapshot_deploy.sh b/snapshot_deploy.sh
index f0af3a9df..a7b5631ff 100755
--- a/snapshot_deploy.sh
+++ b/snapshot_deploy.sh
@@ -1,9 +1,10 @@
 #!/bin/bash -e
 
 THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-export GPG_KEY_ID="Snowflake Computing"
-export SONATYPE_USER="$sonatype_user"
-export SONATYPE_PWD="$sonatype_password"
+if [ -z "$GPG_KEY_ID" ]; then
+  echo "[ERROR] Key Id not specified!"
+  exit 1
+fi
 
 if [ -z "$GPG_KEY_PASSPHRASE" ]; then
   echo "[ERROR] GPG passphrase is not specified for $GPG_KEY_ID!"

From 2d06601c01891ddc1d1a67c46e11decefe261226 Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Fri, 12 May 2023 20:39:39 +0000
Subject: [PATCH 217/356] modified snapshot_deploy.sh file

---
 snapshot_deploy.sh | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/snapshot_deploy.sh b/snapshot_deploy.sh
index a7b5631ff..719e44cfb 100755
--- a/snapshot_deploy.sh
+++ b/snapshot_deploy.sh
@@ -1,10 +1,9 @@
 #!/bin/bash -e
 
 THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-if [ -z "$GPG_KEY_ID" ]; then
-  echo "[ERROR] Key Id not specified!"
-  exit 1
-fi
+export GPG_KEY_ID="Snowflake Computing"
+export SONATYPE_USER="$sonatype_user"
+export SONATYPE_PWD="$sonatype_password"
 
 if [ -z "$GPG_KEY_PASSPHRASE" ]; then
   echo "[ERROR] GPG passphrase is not specified for $GPG_KEY_ID!"
@@ -20,7 +19,6 @@ echo "[INFO] Import PGP Key"
 if ! gpg --list-secret-key | grep "$GPG_KEY_ID"; then
   gpg --allow-secret-key-import --import "$GPG_PRIVATE_KEY"
 fi
-
 # copy the settings.xml template and inject credential information
 SNAPSHOT_DEPLOY_SETTINGS_XML="$THIS_DIR/mvn_settings_snapshot_deploy.xml"
 MVN_REPOSITORY_ID=snapshot

From 35c54a299cd4af17d0fa5c444e6b7df3eb2e1501 Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Fri, 12 May 2023 21:38:57 +0000
Subject: [PATCH 218/356] modifying snapshot_deploy.sh

---
 snapshot_deploy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/snapshot_deploy.sh b/snapshot_deploy.sh
index 719e44cfb..053436071 100755
--- a/snapshot_deploy.sh
+++ b/snapshot_deploy.sh
@@ -53,7 +53,7 @@ echo "[INFO] Close and Release"
 snowflake_repositories=$(mvn ${MVN_OPTIONS[@]} \
     org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-list \
     -DserverId=$MVN_REPOSITORY_ID versions:set -DnewVersion=$project_version-SNAPSHOT
-    -DnexusUrl=https://nexus.int.snowflakecomputing.com/| grep netsnowflake | awk '{print $2}')
+    -DnexusUrl=https://nexus.int.snowflakecomputing.com/ | grep net.snowflake | awk '{print $2}')
 IFS=" "
 if (( $(echo $snowflake_repositories | wc -l)!=1 )); then
     echo "[ERROR] Not single netsnowflake repository is staged. Login https://nexus.int.snowflakecomputing.com/ and make sure no netsnowflake remains there."

From 2f827ae48949da5bb1cbe3e05a8f1a80f6d02d59 Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Fri, 12 May 2023 22:52:37 +0000
Subject: [PATCH 219/356] changed version to snapshot  pom.xml file

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index a823131b8..291b71584 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>1.1.3</version>
+  <version>1.1.3-SNAPSHOT</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>

From 42233291f87a4385356ec5751dc1fdc049fcf025 Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Fri, 12 May 2023 23:24:38 +0000
Subject: [PATCH 220/356] modified snapshot_deploy.sh file

---
 snapshot_deploy.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/snapshot_deploy.sh b/snapshot_deploy.sh
index 053436071..6dfb99961 100755
--- a/snapshot_deploy.sh
+++ b/snapshot_deploy.sh
@@ -52,8 +52,8 @@ mvn deploy ${MVN_OPTIONS[@]} -Dsnapshot-deploy
 echo "[INFO] Close and Release"
 snowflake_repositories=$(mvn ${MVN_OPTIONS[@]} \
     org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-list \
-    -DserverId=$MVN_REPOSITORY_ID versions:set -DnewVersion=$project_version-SNAPSHOT
-    -DnexusUrl=https://nexus.int.snowflakecomputing.com/ | grep net.snowflake | awk '{print $2}')
+    -DserverId=$MVN_REPOSITORY_ID versions:set -DnewVersion=$project_version-SNAPSHOT \
+    -DnexusUrl=https://nexus.int.snowflakecomputing.com/ | grep netsnowflake | awk '{print $2}')
 IFS=" "
 if (( $(echo $snowflake_repositories | wc -l)!=1 )); then
     echo "[ERROR] Not single netsnowflake repository is staged. Login https://nexus.int.snowflakecomputing.com/ and make sure no netsnowflake remains there."

From bf5b79c68b666c9828ae5173df67862ec6802f77 Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Fri, 12 May 2023 23:40:19 +0000
Subject: [PATCH 221/356] updated credentials

---
 snapshot_deploy.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/snapshot_deploy.sh b/snapshot_deploy.sh
index 6dfb99961..0a2a93210 100755
--- a/snapshot_deploy.sh
+++ b/snapshot_deploy.sh
@@ -4,6 +4,8 @@ THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 export GPG_KEY_ID="Snowflake Computing"
 export SONATYPE_USER="$sonatype_user"
 export SONATYPE_PWD="$sonatype_password"
+export LDAP_USER="$sonatype_user"
+export LDAP_PWD="$sonatype_password"
 
 if [ -z "$GPG_KEY_PASSPHRASE" ]; then
   echo "[ERROR] GPG passphrase is not specified for $GPG_KEY_ID!"
@@ -31,8 +33,8 @@ cat > $SNAPSHOT_DEPLOY_SETTINGS_XML << SETTINGS.XML
   <servers>
     <server>
       <id>$MVN_REPOSITORY_ID</id>
-      <username>$SONATYPE_USER</username>
-      <password>$SONATYPE_PWD</password>
+      <username>$LDAP_USER</username>
+      <password>$LDAP_PWD</password>
     </server>
   </servers>
 </settings>

From 3900dab753572a59c07008dd8d6dca1e4f3ff709 Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Sat, 13 May 2023 06:25:23 +0000
Subject: [PATCH 222/356] modified url

---
 snapshot_deploy.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/snapshot_deploy.sh b/snapshot_deploy.sh
index 0a2a93210..77cf7f301 100755
--- a/snapshot_deploy.sh
+++ b/snapshot_deploy.sh
@@ -55,7 +55,7 @@ echo "[INFO] Close and Release"
 snowflake_repositories=$(mvn ${MVN_OPTIONS[@]} \
     org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-list \
     -DserverId=$MVN_REPOSITORY_ID versions:set -DnewVersion=$project_version-SNAPSHOT \
-    -DnexusUrl=https://nexus.int.snowflakecomputing.com/ | grep netsnowflake | awk '{print $2}')
+    -DnexusUrl=https://nexus.int.snowflakecomputing.com/repository/Snapshots/ | grep netsnowflake | awk '{print $2}')
 IFS=" "
 if (( $(echo $snowflake_repositories | wc -l)!=1 )); then
     echo "[ERROR] Not single netsnowflake repository is staged. Login https://nexus.int.snowflakecomputing.com/ and make sure no netsnowflake remains there."
@@ -64,14 +64,14 @@ fi
 if ! mvn ${MVN_OPTIONS[@]} \
     org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-close \
     -DserverId=$MVN_REPOSITORY_ID versions:set -DnewVersion=$project_version-SNAPSHOT\
-    -DnexusUrl=https://nexus.int.snowflakecomputing.com/ \
+    -DnexusUrl=https://nexus.int.snowflakecomputing.com/repository/Snapshots/ \
     -DstagingRepositoryId=$snowflake_repositories \
     -DstagingDescription="Automated Close"; then
     echo "[ERROR] Failed to close. Fix the errors and try this script again"
     mvn ${MVN_OPTIONS[@]} \
         nexus-staging:rc-drop \
         -DserverId=$MVN_REPOSITORY_ID versions:set -DnewVersion=$project_version-SNAPSHOT\
-        -DnexusUrl=https://nexus.int.snowflakecomputing.com/ \
+        -DnexusUrl=https://nexus.int.snowflakecomputing.com/repository/Snapshots/ \
         -DstagingRepositoryId=$snowflake_repositories \
         -DstagingDescription="Failed to close. Dropping..."
 fi
@@ -79,7 +79,7 @@ fi
 mvn ${MVN_OPTIONS[@]} \
     org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-release \
     -DserverId=$MVN_REPOSITORY_ID versions:set -DnewVersion=$project_version-SNAPSHOT\
-    -DnexusUrl=https://nexus.int.snowflakecomputing.com/ \
+    -DnexusUrl=https://nexus.int.snowflakecomputing.com/repository/Snapshots/ \
     -DstagingRepositoryId=$snowflake_repositories \
     -DstagingDescription="Automated Release"
 

From 4d8ab3e9874543a3075786da54266ddac6f90c62 Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Sat, 13 May 2023 06:53:05 +0000
Subject: [PATCH 223/356] addes nexus-staging-plugin

---
 pom.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pom.xml b/pom.xml
index 291b71584..5fb4392fc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -996,6 +996,11 @@
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-gpg-plugin</artifactId>
             <version>1.6</version>
+          </plugin>
+          <plugin>
+            <groupId>org.sonatype.plugins</groupId>
+            <artifactId>nexus-staging-maven-plugin</artifactId>
+            <version>1.6.7</version>
             <executions>
               <execution>
                 <goals>

From a16f0809c9f91cf860a5ba5d204c863ff5c36f83 Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Mon, 15 May 2023 16:57:38 +0000
Subject: [PATCH 224/356] modifiied snapshot version in pom and added unique id
 identifier

---
 pom.xml            | 7 +------
 snapshot_deploy.sh | 2 +-
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/pom.xml b/pom.xml
index 5fb4392fc..a2a3eab66 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>1.1.3-SNAPSHOT</version>
+  <version>1.1.4-SNAPSHOT</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
@@ -996,11 +996,6 @@
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-gpg-plugin</artifactId>
             <version>1.6</version>
-          </plugin>
-          <plugin>
-            <groupId>org.sonatype.plugins</groupId>
-            <artifactId>nexus-staging-maven-plugin</artifactId>
-            <version>1.6.7</version>
             <executions>
               <execution>
                 <goals>
diff --git a/snapshot_deploy.sh b/snapshot_deploy.sh
index 77cf7f301..e51e3012a 100755
--- a/snapshot_deploy.sh
+++ b/snapshot_deploy.sh
@@ -49,7 +49,7 @@ echo "[Info] Sign package and deploy to staging area"
 project_version=$($THIS_DIR/scripts/get_project_info_from_pom.py $THIS_DIR/pom.xml version)
 $THIS_DIR/scripts/update_project_version.py public_pom.xml $project_version > generated_public_pom.xml
 
-mvn deploy ${MVN_OPTIONS[@]} -Dsnapshot-deploy
+mvn deploy ${MVN_OPTIONS[@]} -Dsnapshot-deploy -DuniqueVersion=false
 
 echo "[INFO] Close and Release"
 snowflake_repositories=$(mvn ${MVN_OPTIONS[@]} \

From 812dea1a89c06a1723ee6d05269d1d28892dc289 Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Mon, 15 May 2023 17:11:03 +0000
Subject: [PATCH 225/356] adding uniqueVersion parameter false

---
 pom.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pom.xml b/pom.xml
index a2a3eab66..5c87b001b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1007,6 +1007,7 @@
                   <repositoryId>snapshot</repositoryId>
                   <url>https://nexus.int.snowflakecomputing.com/repository/Snapshots/</url>
                   <pomFile>generated_public_pom.xml</pomFile>
+                  <uniqueVersion>false</uniqueVersion>
                   <javadoc>target/${project.artifactId}-javadoc.jar</javadoc>
                   <sources>target/${project.artifactId}-sources.jar</sources>
                   <keyname>${env.GPG_KEY_ID}</keyname>

From d0fa026fd307d8db230a99e4677d9133a8932286 Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Mon, 15 May 2023 20:10:17 +0000
Subject: [PATCH 226/356] remove uniqueVersion parameter

---
 pom.xml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 5c87b001b..a2a3eab66 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1007,7 +1007,6 @@
                   <repositoryId>snapshot</repositoryId>
                   <url>https://nexus.int.snowflakecomputing.com/repository/Snapshots/</url>
                   <pomFile>generated_public_pom.xml</pomFile>
-                  <uniqueVersion>false</uniqueVersion>
                   <javadoc>target/${project.artifactId}-javadoc.jar</javadoc>
                   <sources>target/${project.artifactId}-sources.jar</sources>
                   <keyname>${env.GPG_KEY_ID}</keyname>

From 48a805865a3d7f997249e09273f6f6e0b1c2c137 Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Wed, 17 May 2023 22:30:31 +0000
Subject: [PATCH 227/356] removed build version from snapshot_deploy.sh

---
 snapshot_deploy.sh | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/snapshot_deploy.sh b/snapshot_deploy.sh
index e51e3012a..b43a51a1c 100755
--- a/snapshot_deploy.sh
+++ b/snapshot_deploy.sh
@@ -2,8 +2,6 @@
 
 THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 export GPG_KEY_ID="Snowflake Computing"
-export SONATYPE_USER="$sonatype_user"
-export SONATYPE_PWD="$sonatype_password"
 export LDAP_USER="$sonatype_user"
 export LDAP_PWD="$sonatype_password"
 
@@ -49,13 +47,13 @@ echo "[Info] Sign package and deploy to staging area"
 project_version=$($THIS_DIR/scripts/get_project_info_from_pom.py $THIS_DIR/pom.xml version)
 $THIS_DIR/scripts/update_project_version.py public_pom.xml $project_version > generated_public_pom.xml
 
-mvn deploy ${MVN_OPTIONS[@]} -Dsnapshot-deploy -DuniqueVersion=false
+mvn clean deploy ${MVN_OPTIONS[@]} -Dsnapshot-deploy
 
 echo "[INFO] Close and Release"
 snowflake_repositories=$(mvn ${MVN_OPTIONS[@]} \
     org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-list \
-    -DserverId=$MVN_REPOSITORY_ID versions:set -DnewVersion=$project_version-SNAPSHOT \
-    -DnexusUrl=https://nexus.int.snowflakecomputing.com/repository/Snapshots/ | grep netsnowflake | awk '{print $2}')
+    -DserverId=$MVN_REPOSITORY_ID  \
+    -DnexusUrl=https://nexus.int.snowflakecomputing.com/ | grep netsnowflake | awk '{print $2}')
 IFS=" "
 if (( $(echo $snowflake_repositories | wc -l)!=1 )); then
     echo "[ERROR] Not single netsnowflake repository is staged. Login https://nexus.int.snowflakecomputing.com/ and make sure no netsnowflake remains there."
@@ -63,23 +61,23 @@ if (( $(echo $snowflake_repositories | wc -l)!=1 )); then
 fi
 if ! mvn ${MVN_OPTIONS[@]} \
     org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-close \
-    -DserverId=$MVN_REPOSITORY_ID versions:set -DnewVersion=$project_version-SNAPSHOT\
-    -DnexusUrl=https://nexus.int.snowflakecomputing.com/repository/Snapshots/ \
+    -DserverId=$MVN_REPOSITORY_ID \
+    -DnexusUrl=https://nexus.int.snowflakecomputing.com/  \
     -DstagingRepositoryId=$snowflake_repositories \
     -DstagingDescription="Automated Close"; then
     echo "[ERROR] Failed to close. Fix the errors and try this script again"
     mvn ${MVN_OPTIONS[@]} \
         nexus-staging:rc-drop \
-        -DserverId=$MVN_REPOSITORY_ID versions:set -DnewVersion=$project_version-SNAPSHOT\
-        -DnexusUrl=https://nexus.int.snowflakecomputing.com/repository/Snapshots/ \
+        -DserverId=$MVN_REPOSITORY_ID \
+        -DnexusUrl=https://nexus.int.snowflakecomputing.com/ \
         -DstagingRepositoryId=$snowflake_repositories \
         -DstagingDescription="Failed to close. Dropping..."
 fi
 
 mvn ${MVN_OPTIONS[@]} \
     org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-release \
-    -DserverId=$MVN_REPOSITORY_ID versions:set -DnewVersion=$project_version-SNAPSHOT\
-    -DnexusUrl=https://nexus.int.snowflakecomputing.com/repository/Snapshots/ \
+    -DserverId=$MVN_REPOSITORY_ID \
+    -DnexusUrl=https://nexus.int.snowflakecomputing.com/ \
     -DstagingRepositoryId=$snowflake_repositories \
     -DstagingDescription="Automated Release"
 

From 9a8535d655dd08ced8b11a9be785ee033f87f508 Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Wed, 17 May 2023 22:45:56 +0000
Subject: [PATCH 228/356] test

---
 snapshot_deploy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/snapshot_deploy.sh b/snapshot_deploy.sh
index b43a51a1c..6d4229156 100755
--- a/snapshot_deploy.sh
+++ b/snapshot_deploy.sh
@@ -56,7 +56,7 @@ snowflake_repositories=$(mvn ${MVN_OPTIONS[@]} \
     -DnexusUrl=https://nexus.int.snowflakecomputing.com/ | grep netsnowflake | awk '{print $2}')
 IFS=" "
 if (( $(echo $snowflake_repositories | wc -l)!=1 )); then
-    echo "[ERROR] Not single netsnowflake repository is staged. Login https://nexus.int.snowflakecomputing.com/ and make sure no netsnowflake remains there."
+    echo "[ERROR] Not single netsnowflake repository is staged.Login https://nexus.int.snowflakecomputing.com/ and make sure no netsnowflake remains there."
     exit 1
 fi
 if ! mvn ${MVN_OPTIONS[@]} \

From 2e0d614501a371b574f8da66ce393fb1e9e061f5 Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Wed, 17 May 2023 23:24:56 +0000
Subject: [PATCH 229/356] changed the plugin version

---
 snapshot_deploy.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/snapshot_deploy.sh b/snapshot_deploy.sh
index 6d4229156..d5140f276 100755
--- a/snapshot_deploy.sh
+++ b/snapshot_deploy.sh
@@ -51,7 +51,7 @@ mvn clean deploy ${MVN_OPTIONS[@]} -Dsnapshot-deploy
 
 echo "[INFO] Close and Release"
 snowflake_repositories=$(mvn ${MVN_OPTIONS[@]} \
-    org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-list \
+    org.sonatype.plugins:nexus-staging-maven-plugin:1.6.13:rc-list \
     -DserverId=$MVN_REPOSITORY_ID  \
     -DnexusUrl=https://nexus.int.snowflakecomputing.com/ | grep netsnowflake | awk '{print $2}')
 IFS=" "
@@ -60,7 +60,7 @@ if (( $(echo $snowflake_repositories | wc -l)!=1 )); then
     exit 1
 fi
 if ! mvn ${MVN_OPTIONS[@]} \
-    org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-close \
+    org.sonatype.plugins:nexus-staging-maven-plugin:1.6.13:rc-close \
     -DserverId=$MVN_REPOSITORY_ID \
     -DnexusUrl=https://nexus.int.snowflakecomputing.com/  \
     -DstagingRepositoryId=$snowflake_repositories \
@@ -75,7 +75,7 @@ if ! mvn ${MVN_OPTIONS[@]} \
 fi
 
 mvn ${MVN_OPTIONS[@]} \
-    org.sonatype.plugins:nexus-staging-maven-plugin:1.6.7:rc-release \
+    org.sonatype.plugins:nexus-staging-maven-plugin:1.6.13:rc-release \
     -DserverId=$MVN_REPOSITORY_ID \
     -DnexusUrl=https://nexus.int.snowflakecomputing.com/ \
     -DstagingRepositoryId=$snowflake_repositories \

From 63ebcf6f75d665bb91969586a2b07b67c04b640b Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Fri, 19 May 2023 16:25:04 +0000
Subject: [PATCH 230/356] removed staging code

---
 snapshot_deploy.sh | 32 --------------------------------
 1 file changed, 32 deletions(-)

diff --git a/snapshot_deploy.sh b/snapshot_deploy.sh
index d5140f276..05fbaf4e3 100755
--- a/snapshot_deploy.sh
+++ b/snapshot_deploy.sh
@@ -49,36 +49,4 @@ $THIS_DIR/scripts/update_project_version.py public_pom.xml $project_version > ge
 
 mvn clean deploy ${MVN_OPTIONS[@]} -Dsnapshot-deploy
 
-echo "[INFO] Close and Release"
-snowflake_repositories=$(mvn ${MVN_OPTIONS[@]} \
-    org.sonatype.plugins:nexus-staging-maven-plugin:1.6.13:rc-list \
-    -DserverId=$MVN_REPOSITORY_ID  \
-    -DnexusUrl=https://nexus.int.snowflakecomputing.com/ | grep netsnowflake | awk '{print $2}')
-IFS=" "
-if (( $(echo $snowflake_repositories | wc -l)!=1 )); then
-    echo "[ERROR] Not single netsnowflake repository is staged.Login https://nexus.int.snowflakecomputing.com/ and make sure no netsnowflake remains there."
-    exit 1
-fi
-if ! mvn ${MVN_OPTIONS[@]} \
-    org.sonatype.plugins:nexus-staging-maven-plugin:1.6.13:rc-close \
-    -DserverId=$MVN_REPOSITORY_ID \
-    -DnexusUrl=https://nexus.int.snowflakecomputing.com/  \
-    -DstagingRepositoryId=$snowflake_repositories \
-    -DstagingDescription="Automated Close"; then
-    echo "[ERROR] Failed to close. Fix the errors and try this script again"
-    mvn ${MVN_OPTIONS[@]} \
-        nexus-staging:rc-drop \
-        -DserverId=$MVN_REPOSITORY_ID \
-        -DnexusUrl=https://nexus.int.snowflakecomputing.com/ \
-        -DstagingRepositoryId=$snowflake_repositories \
-        -DstagingDescription="Failed to close. Dropping..."
-fi
-
-mvn ${MVN_OPTIONS[@]} \
-    org.sonatype.plugins:nexus-staging-maven-plugin:1.6.13:rc-release \
-    -DserverId=$MVN_REPOSITORY_ID \
-    -DnexusUrl=https://nexus.int.snowflakecomputing.com/ \
-    -DstagingRepositoryId=$snowflake_repositories \
-    -DstagingDescription="Automated Release"
-
 rm $SNAPSHOT_DEPLOY_SETTINGS_XML
\ No newline at end of file

From cf8e9b7e13d1b30123c689fd728f56e309adc9b2 Mon Sep 17 00:00:00 2001
From: Dhara <dhara.shah@snowflake.com>
Date: Fri, 19 May 2023 19:48:56 +0000
Subject: [PATCH 231/356] updated version in pom.xml file and RequestBuilder
 file

---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index a2a3eab66..12c52c34c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>1.1.4-SNAPSHOT</version>
+  <version>1.1.5-SNAPSHOT</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index b3b2b5947..47e1574b5 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -127,7 +127,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.1.3";
+  public static final String DEFAULT_VERSION = "1.1.5-SNAPSHOT";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 210215c55f29adcbfd502c872dc39aca680b75c6 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Mon, 22 May 2023 18:39:16 +0200
Subject: [PATCH 232/356] Revert "SNOW-811371 Shade and Relocate Snowflake JDBC
 Classes (#494)" (#503)

This reverts commit 5ae0dcb20faa2b5d51d980bf9008d8f87642891b.
---
 pom.xml | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/pom.xml b/pom.xml
index 12c52c34c..eddd2b12c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -794,19 +794,16 @@
             <artifactId>maven-shade-plugin</artifactId>
             <version>3.4.1</version>
             <configuration>
+              <artifactSet>
+                <excludes>
+                  <exclude>net.snowflake:snowflake-jdbc</exclude>
+                </excludes>
+              </artifactSet>
               <relocations>
                 <relocation>
                   <pattern>com.nimbusds</pattern>
                   <shadedPattern>${shadeBase}.com.nimbusds</shadedPattern>
                 </relocation>
-                <relocation>
-                  <pattern>net.snowflake.client</pattern>
-                  <shadedPattern>${shadeBase}.net.snowflake.client</shadedPattern>
-                </relocation>
-                <relocation>
-                  <pattern>com.snowflake.client</pattern>
-                  <shadedPattern>${shadeBase}.com.snowflake.client</shadedPattern>
-                </relocation>
                 <relocation>
                   <pattern>net.jcip</pattern>
                   <shadedPattern>${shadeBase}.net.jcip</shadedPattern>

From e855d8df7999ae1ee4680295776f86d13e778ce1 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Tue, 23 May 2023 10:18:13 +0200
Subject: [PATCH 233/356] Fix 'Connection reset' build errors by caching mvn
 dependencies (#502)

---
 .github/workflows/End2EndTest.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/End2EndTest.yml b/.github/workflows/End2EndTest.yml
index 12b144c76..a98c97fd2 100644
--- a/.github/workflows/End2EndTest.yml
+++ b/.github/workflows/End2EndTest.yml
@@ -19,9 +19,11 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v2
       - name: Install Java ${{ matrix.java }}
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v2
         with:
+          distribution: temurin
           java-version: ${{ matrix.java }}
+          cache: maven
 
       - name: Decrypt profile.json for Cloud ${{ matrix.snowflake_cloud }}
         env:
@@ -51,9 +53,11 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v2
       - name: Install Java ${{ matrix.java }}
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v2
         with:
+          distribution: temurin
           java-version: ${{ matrix.java }}
+          cache: maven
       - name: Decrypt profile.json for Cloud ${{ matrix.snowflake_cloud }} on Windows Powershell
         env:
           DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }}

From 36ee832c53e949bf6b0afb89cfd7d90a6085b6d2 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Wed, 24 May 2023 12:28:08 +0200
Subject: [PATCH 234/356] SNOW-768197 Attribution notice (#506)

This PR adds licenses of SDK dependencies to the META-INF directory of the fat
jar. In pom.xml, we maintain the list of allowed licenses. In the 'package'
build phase, we verify there aren't any new dependencies, not mentioned in the
allowed list.
---
 pom.xml                                       |  44 ++++
 .../third-party-licenses/Apache2.0.txt        | 201 ++++++++++++++++++
 .../third-party-licenses/BSD-2-Clause.txt     |  24 +++
 .../third-party-licenses/BSD-3-Clause.txt     |  28 +++
 .../META-INF/third-party-licenses/EDL1.0.txt  |  12 ++
 .../META-INF/third-party-licenses/Go.txt      |  27 +++
 .../META-INF/third-party-licenses/MIT.txt     |  21 ++
 .../third-party-licenses/REVISED_BSD.txt      |  30 +++
 8 files changed, 387 insertions(+)
 create mode 100644 src/main/resources/META-INF/third-party-licenses/Apache2.0.txt
 create mode 100644 src/main/resources/META-INF/third-party-licenses/BSD-2-Clause.txt
 create mode 100644 src/main/resources/META-INF/third-party-licenses/BSD-3-Clause.txt
 create mode 100644 src/main/resources/META-INF/third-party-licenses/EDL1.0.txt
 create mode 100644 src/main/resources/META-INF/third-party-licenses/Go.txt
 create mode 100644 src/main/resources/META-INF/third-party-licenses/MIT.txt
 create mode 100644 src/main/resources/META-INF/third-party-licenses/REVISED_BSD.txt

diff --git a/pom.xml b/pom.xml
index eddd2b12c..c0cc3b4ca 100644
--- a/pom.xml
+++ b/pom.xml
@@ -742,6 +742,50 @@
           </excludes>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>license-maven-plugin</artifactId>
+        <version>2.0.1</version>
+        <configuration>
+          <errorRemedy>failFast</errorRemedy>
+          <includedLicenses>
+            <includedLicense>Apache License 2.0</includedLicense>
+            <includedLicense>BSD 2-Clause License</includedLicense>
+            <includedLicense>3-Clause BSD License</includedLicense>
+            <includedLicense>Revised BSD</includedLicense>
+            <includedLicense>The MIT License</includedLicense>
+            <includedLicense>EDL 1.0</includedLicense>
+            <includedLicense>The Go license</includedLicense>
+          </includedLicenses>
+          <!--
+          The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please
+          check your dependencies", verify the conditions of the license, put the text of the license into third-party-licenses/
+          directory and add it to this list. You can also run "mvn license:third-party-report" to get an HTML license report.
+          -->
+          <excludedScopes>test,provided,system</excludedScopes>
+          <failOnBlacklist>true</failOnBlacklist>
+          <licenseMerges>
+            <licenseMerge>Apache License 2.0
+              |The Apache License, Version 2.0
+              |The Apache Software License, Version 2.0
+              |Apache-2.0
+              |Apache License, Version 2.0
+              |Apache 2.0
+              |Apache License V2.0</licenseMerge>
+            <licenseMerge>BSD 2-Clause License
+              |The BSD License</licenseMerge>
+          </licenseMerges>
+        </configuration>
+        <executions>
+          <execution>
+            <id>add-third-party</id>
+            <goals>
+              <goal>add-third-party</goal>
+            </goals>
+            <phase>package</phase>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 
diff --git a/src/main/resources/META-INF/third-party-licenses/Apache2.0.txt b/src/main/resources/META-INF/third-party-licenses/Apache2.0.txt
new file mode 100644
index 000000000..f49a4e16e
--- /dev/null
+++ b/src/main/resources/META-INF/third-party-licenses/Apache2.0.txt
@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
\ No newline at end of file
diff --git a/src/main/resources/META-INF/third-party-licenses/BSD-2-Clause.txt b/src/main/resources/META-INF/third-party-licenses/BSD-2-Clause.txt
new file mode 100644
index 000000000..805473dfd
--- /dev/null
+++ b/src/main/resources/META-INF/third-party-licenses/BSD-2-Clause.txt
@@ -0,0 +1,24 @@
+BSD 2-Clause License
+
+Copyright (c) [year], [fullname]
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/src/main/resources/META-INF/third-party-licenses/BSD-3-Clause.txt b/src/main/resources/META-INF/third-party-licenses/BSD-3-Clause.txt
new file mode 100644
index 000000000..ae5698f13
--- /dev/null
+++ b/src/main/resources/META-INF/third-party-licenses/BSD-3-Clause.txt
@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) [year], [fullname]
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/src/main/resources/META-INF/third-party-licenses/EDL1.0.txt b/src/main/resources/META-INF/third-party-licenses/EDL1.0.txt
new file mode 100644
index 000000000..a4a267ef0
--- /dev/null
+++ b/src/main/resources/META-INF/third-party-licenses/EDL1.0.txt
@@ -0,0 +1,12 @@
+Eclipse Distribution License - v 1.0
+
+Copyright (c) 2007, Eclipse Foundation, Inc. and its licensors.
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+Neither the name of the Eclipse Foundation, Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/src/main/resources/META-INF/third-party-licenses/Go.txt b/src/main/resources/META-INF/third-party-licenses/Go.txt
new file mode 100644
index 000000000..6a66aea5e
--- /dev/null
+++ b/src/main/resources/META-INF/third-party-licenses/Go.txt
@@ -0,0 +1,27 @@
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/src/main/resources/META-INF/third-party-licenses/MIT.txt b/src/main/resources/META-INF/third-party-licenses/MIT.txt
new file mode 100644
index 000000000..63b4b681c
--- /dev/null
+++ b/src/main/resources/META-INF/third-party-licenses/MIT.txt
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [year] [fullname]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
\ No newline at end of file
diff --git a/src/main/resources/META-INF/third-party-licenses/REVISED_BSD.txt b/src/main/resources/META-INF/third-party-licenses/REVISED_BSD.txt
new file mode 100644
index 000000000..641d4f45e
--- /dev/null
+++ b/src/main/resources/META-INF/third-party-licenses/REVISED_BSD.txt
@@ -0,0 +1,30 @@
+JSch 0.0.* was released under the GNU LGPL license.  Later, we have switched
+over to a BSD-style license.
+
+------------------------------------------------------------------------------
+Copyright (c) 2002-2015 Atsuhiko Yamanaka, JCraft,Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+  1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+
+  2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in
+     the documentation and/or other materials provided with the distribution.
+
+  3. The names of the authors may not be used to endorse or promote products
+     derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT,
+INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file

From c24d3069e5ef57d341a06b6ca9f9f6e3300bd142 Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Tue, 23 May 2023 10:07:25 +0000
Subject: [PATCH 235/356] @snow SNOW-763199 Enable
 StreamingIngestIT.testTableColumnEvolution

---
 .../snowflake/ingest/streaming/internal/StreamingIngestIT.java  | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 4be4b254e..1f9ea3914 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -46,7 +46,6 @@
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -1083,7 +1082,6 @@ public void testColumnNameQuotes() throws Exception {
     Assert.fail("Row sequencer not updated before timeout");
   }
 
-  @Ignore
   @Test
   public void testTableColumnEvolution() throws Exception {
     final int rowNum = 100;

From a3e2bf8444c59d10c17fe8d63e075bf2e5344ca4 Mon Sep 17 00:00:00 2001
From: Xin Huang <xin.huang@snowflake.com>
Date: Thu, 25 May 2023 08:08:20 -0700
Subject: [PATCH 236/356] minor error text typo fix

---
 .../net/snowflake/ingest/ingest_error_messages.properties       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
index 1008bcb8b..5aaeadfba 100644
--- a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
+++ b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
@@ -30,6 +30,6 @@
 0025=One or more channels {0} might contain uncommitted rows due to server side errors, please consider reopening the channels to replay the data loading by using the latest persistent offset token.
 0026=Invalid collation string: {0}. {1}
 0027=Failure during data encryption.
-0028=Get channel status indicates Channel {0} is invalid with status code {1}, please reopening the channel.
+0028=Get channel status indicates Channel {0} is invalid with status code {1}, please reopen the channel.
 0029=Data type not supported: {0}
 0030=The given row cannot be converted to the internal format due to invalid value: {0}

From f24f4314cd5e333d84670851df5629763385ebf5 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 25 May 2023 23:41:08 -0700
Subject: [PATCH 237/356] SNOW-768195: [Client Side] Support GCS downscoped
 token for Snowpipe Streaming (#504)

SNOW-768195

[Client Side] Support GCS downscoped token for Snowpipe Streaming
---
 pom.xml                                                       | 2 +-
 .../snowflake/ingest/streaming/internal/ParquetRowBuffer.java | 1 +
 .../ingest/streaming/internal/StreamingIngestStage.java       | 4 ++--
 .../java/org/apache/parquet/hadoop/BdecParquetWriter.java     | 1 -
 .../ingest/streaming/internal/StreamingIngestStageTest.java   | 4 ++--
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/pom.xml b/pom.xml
index c0cc3b4ca..1be577462 100644
--- a/pom.xml
+++ b/pom.xml
@@ -61,7 +61,7 @@
     <shadeBase>net.snowflake.ingest.internal</shadeBase>
     <slf4j.version>1.7.36</slf4j.version>
     <snappy.version>1.1.8.3</snappy.version>
-    <snowjdbc.version>3.13.29</snowjdbc.version>
+    <snowjdbc.version>3.13.30</snowjdbc.version>
     <yetus.version>0.13.0</yetus.version>
   </properties>
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 73774d080..9c35a13df 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -43,6 +43,7 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
 
   /* Unflushed rows as Java objects. Needed for the Parquet w/o memory optimization. */
   private final List<List<Object>> data;
+
   /* BDEC Parquet writer. It is used to buffer unflushed data in Parquet internal buffers instead of using Java objects */
   private BdecParquetWriter bdecParquetWriter;
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index da1662e51..2feed6b81 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -146,8 +146,7 @@ void putRemote(String fullFilePath, byte[] data) throws SnowflakeSQLException, I
   private void putRemote(String fullFilePath, byte[] data, int retryCount)
       throws SnowflakeSQLException, IOException {
     SnowflakeFileTransferMetadataV1 fileTransferMetadataCopy;
-    if (this.fileTransferMetadataWithAge.fileTransferMetadata.getStageInfo().getStageType()
-        == StageInfo.StageType.GCS) {
+    if (this.fileTransferMetadataWithAge.fileTransferMetadata.isForOneFile()) {
       fileTransferMetadataCopy = this.fetchSignedURL(fullFilePath);
     } else {
       // Set file path to be uploaded
@@ -186,6 +185,7 @@ private void putRemote(String fullFilePath, byte[] data, int retryCount)
               .setStreamingIngestClientKey(this.clientPrefix)
               .setStreamingIngestClientName(this.clientName)
               .setProxyProperties(this.proxyProperties)
+              .setDestFileName(fullFilePath)
               .build());
     } catch (SnowflakeSQLException e) {
       if (e.getErrorCode() != CLOUD_STORAGE_CREDENTIALS_EXPIRED || retryCount >= MAX_RETRY_COUNT) {
diff --git a/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java b/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java
index 55ab1bf30..b2442d3dc 100644
--- a/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java
+++ b/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java
@@ -148,7 +148,6 @@ private static ParquetProperties createParquetProperties() {
         // server side does not support it TODO: SNOW-657238
         .withWriterVersion(ParquetProperties.WriterVersion.PARQUET_1_0)
         .withValuesWriterFactory(new DefaultV1ValuesWriterFactory())
-
         // the dictionary encoding (Encoding.*_DICTIONARY) is not supported by server side
         // scanner yet
         .withDictionaryEncoding(false)
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
index bdbe42df8..a137ab9ed 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
@@ -80,8 +80,8 @@ public class StreamingIngestStageTest {
           + " \"region\": \"us-east-1\", \"storageAccount\": null, \"isClientSideEncrypted\":"
           + " true, \"creds\": {\"AWS_KEY_ID\": \"EXAMPLE_AWS_KEY_ID\", \"AWS_SECRET_KEY\":"
           + " \"EXAMPLE_AWS_SECRET_KEY\", \"AWS_TOKEN\": \"EXAMPLE_AWS_TOKEN\", \"AWS_ID\":"
-          + " \"EXAMPLE_AWS_ID\", \"AWS_KEY\": \"EXAMPLE_AWS_KEY\"}, \"presignedUrl\": null,"
-          + " \"endPoint\": null}}}";
+          + " \"EXAMPLE_AWS_ID\", \"AWS_KEY\": \"EXAMPLE_AWS_KEY\"}, \"presignedUrl\":"
+          + " \"presignedUrl\", \"endPoint\": null}}}";
 
   String exampleRemoteMetaResponse =
       "{\"src_locations\": [\"foo/\"],"

From 7ad9119ec4544b79e4f5310f77df30818f87bd27 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Wed, 31 May 2023 13:51:46 +0200
Subject: [PATCH 238/356] SNOW-818891 Remove Arrow and other unneeded
 dependencies (#497)

---
 pom.xml                                       |  39 +-
 .../streaming/internal/AbstractRowBuffer.java |  46 +-
 .../streaming/internal/ArrowFlusher.java      | 121 ---
 .../streaming/internal/ArrowRowBuffer.java    | 813 ------------------
 .../streaming/internal/BlobBuilder.java       | 113 +--
 .../streaming/internal/BlobMetadata.java      |   2 +-
 .../streaming/internal/ChannelCache.java      |   3 +-
 .../streaming/internal/ChannelData.java       |   3 +-
 .../streaming/internal/FlushService.java      |   9 +-
 .../ingest/streaming/internal/Flusher.java    |   3 +-
 .../streaming/internal/ParquetRowBuffer.java  |   3 -
 .../streaming/internal/RegisterService.java   |   3 +-
 .../ingest/streaming/internal/RowBuffer.java  |   3 +-
 ...nowflakeStreamingIngestChannelFactory.java |  17 +-
 ...owflakeStreamingIngestChannelInternal.java |  12 +-
 ...nowflakeStreamingIngestClientInternal.java |  21 +-
 .../internal/StreamingIngestStage.java        |   2 +-
 .../streaming/internal/TimestampWrapper.java  |   2 +-
 .../net/snowflake/ingest/utils/Constants.java |   6 +-
 .../net/snowflake/ingest/utils/Cryptor.java   |   2 +-
 .../ingest/utils/ParameterProvider.java       |   2 +-
 .../net/snowflake/ingest/utils/Utils.java     |   9 -
 .../third-party-licenses/REVISED_BSD.txt      |  30 -
 .../java/net/snowflake/ingest/TestUtils.java  |  10 -
 .../streaming/internal/ArrowBufferTest.java   | 444 ----------
 .../streaming/internal/FlushServiceTest.java  |  72 +-
 .../internal/OpenManyChannelsIT.java          |   2 +-
 .../streaming/internal/RowBufferTest.java     |  87 +-
 .../SnowflakeStreamingIngestChannelTest.java  |  16 +-
 .../SnowflakeStreamingIngestClientTest.java   |  24 +-
 .../internal/StreamingIngestBigFilesIT.java   |  17 +-
 .../streaming/internal/StreamingIngestIT.java |  18 +-
 .../datatypes/AbstractDataTypeTest.java       |  20 +-
 .../internal/datatypes/BinaryIT.java          |   5 -
 .../internal/datatypes/DateTimeIT.java        |  13 +-
 .../internal/datatypes/LogicalTypesIT.java    |   5 -
 .../streaming/internal/datatypes/NullIT.java  |   5 -
 .../internal/datatypes/NumericTypesIT.java    |   5 -
 .../internal/datatypes/SemiStructuredIT.java  |   5 -
 .../internal/datatypes/StringsIT.java         |  15 +-
 .../streaming/internal/it/ColumnNamesIT.java  |   5 -
 41 files changed, 112 insertions(+), 1920 deletions(-)
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
 delete mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
 delete mode 100644 src/main/resources/META-INF/third-party-licenses/REVISED_BSD.txt
 delete mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java

diff --git a/pom.xml b/pom.xml
index 1be577462..dd972a28c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -35,7 +35,6 @@
 
   <!-- Set our Language Level to Java 8 -->
   <properties>
-    <arrow.version>10.0.0</arrow.version>
     <codehaus.version>1.9.13</codehaus.version>
     <commonscodec.version>1.15</commonscodec.version>
     <commonscollections.version>3.2.2</commonscollections.version>
@@ -157,6 +156,10 @@
             <groupId>com.github.pjfanning</groupId>
             <artifactId>jersey-json</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>com.jcraft</groupId>
+            <artifactId>jsch</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>com.sun.jersey</groupId>
             <artifactId>jersey-core</artifactId>
@@ -169,6 +172,10 @@
             <groupId>com.sun.jersey</groupId>
             <artifactId>jersey-servlet</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>dnsjava</groupId>
             <artifactId>dnsjava</artifactId>
@@ -189,10 +196,22 @@
             <groupId>org.apache.avro</groupId>
             <artifactId>avro</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-client</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-recipes</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-auth</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>org.apache.kerby</groupId>
             <artifactId>kerb-core</artifactId>
@@ -406,17 +425,6 @@
       <groupId>net.snowflake</groupId>
       <artifactId>snowflake-jdbc</artifactId>
     </dependency>
-    <!-- Apache Arrow -->
-    <dependency>
-      <groupId>org.apache.arrow</groupId>
-      <artifactId>arrow-memory-core</artifactId>
-      <version>${arrow.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.arrow</groupId>
-      <artifactId>arrow-vector</artifactId>
-      <version>${arrow.version}</version>
-    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
@@ -439,12 +447,6 @@
       <artifactId>slf4j-api</artifactId>
       <scope>provided</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.arrow</groupId>
-      <artifactId>arrow-memory-netty</artifactId>
-      <version>${arrow.version}</version>
-      <scope>runtime</scope>
-    </dependency>
     <!-- JUnit so that we can make some basic unit tests -->
     <dependency>
       <groupId>junit</groupId>
@@ -752,7 +754,6 @@
             <includedLicense>Apache License 2.0</includedLicense>
             <includedLicense>BSD 2-Clause License</includedLicense>
             <includedLicense>3-Clause BSD License</includedLicense>
-            <includedLicense>Revised BSD</includedLicense>
             <includedLicense>The MIT License</includedLicense>
             <includedLicense>EDL 1.0</includedLicense>
             <includedLicense>The Go license</includedLicense>
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 0580006d0..c5d0bddac 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -4,6 +4,7 @@
 
 package net.snowflake.ingest.streaming.internal;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -23,17 +24,13 @@
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.Pair;
 import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.util.VisibleForTesting;
 
 /**
  * The abstract implementation of the buffer in the Streaming Ingest channel that holds the
  * un-flushed rows, these rows will be converted to the underlying format implementation for faster
  * processing
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or Parquet
- *     {@link ParquetChunkData})
+ * @param <T> type of column data ({@link ParquetChunkData} for Parquet)
  */
 abstract class AbstractRowBuffer<T> implements RowBuffer<T> {
   private static final Logging logger = new Logging(AbstractRowBuffer.class);
@@ -162,9 +159,6 @@ public int getOrdinal() {
   // Metric callback to report size of inserted rows
   private final Consumer<Float> rowSizeMetric;
 
-  // Allocator used to allocate the buffers
-  final BufferAllocator allocator;
-
   // State of the owning channel
   final ChannelRuntimeState channelState;
 
@@ -176,7 +170,6 @@ public int getOrdinal() {
   AbstractRowBuffer(
       OpenChannelRequest.OnErrorOption onErrorOption,
       ZoneId defaultTimezone,
-      BufferAllocator allocator,
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelRuntimeState) {
@@ -185,7 +178,6 @@ public int getOrdinal() {
     this.rowSizeMetric = rowSizeMetric;
     this.channelState = channelRuntimeState;
     this.channelFullyQualifiedName = fullyQualifiedChannelName;
-    this.allocator = allocator;
     this.nonNullableFieldNames = new HashSet<>();
     this.flushLock = new ReentrantLock();
     this.bufferedRowCount = 0;
@@ -505,30 +497,8 @@ void reset() {
   /** Close the row buffer and release allocated memory for the channel. */
   @Override
   public synchronized void close(String name) {
-    long allocatedBeforeRelease = this.allocator.getAllocatedMemory();
 
     closeInternal();
-
-    long allocatedAfterRelease = this.allocator.getAllocatedMemory();
-    logger.logInfo(
-        "Trying to close {} for channel={} from function={}, allocatedBeforeRelease={},"
-            + " allocatedAfterRelease={}",
-        this.getClass().getSimpleName(),
-        channelFullyQualifiedName,
-        name,
-        allocatedBeforeRelease,
-        allocatedAfterRelease);
-    Utils.closeAllocator(this.allocator);
-
-    // If the channel is valid but still has leftover data, throw an exception because it should be
-    // cleaned up already before calling close
-    if (allocatedBeforeRelease > 0 && this.channelState.isValid()) {
-      throw new SFException(
-          ErrorCode.INTERNAL_ERROR,
-          String.format(
-              "Memory leaked=%d by allocator=%s, channel=%s",
-              allocatedBeforeRelease, this.allocator, channelFullyQualifiedName));
-    }
   }
 
   /**
@@ -554,7 +524,6 @@ static EpInfo buildEpInfoFromStats(long rowCount, Map<String, RowBufferStats> co
   static <T> AbstractRowBuffer<T> createRowBuffer(
       OpenChannelRequest.OnErrorOption onErrorOption,
       ZoneId defaultTimezone,
-      BufferAllocator allocator,
       Constants.BdecVersion bdecVersion,
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
@@ -562,23 +531,12 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
       boolean enableParquetMemoryOptimization,
       long maxChunkSizeInBytes) {
     switch (bdecVersion) {
-      case ONE:
-        //noinspection unchecked
-        return (AbstractRowBuffer<T>)
-            new ArrowRowBuffer(
-                onErrorOption,
-                defaultTimezone,
-                allocator,
-                fullyQualifiedChannelName,
-                rowSizeMetric,
-                channelRuntimeState);
       case THREE:
         //noinspection unchecked
         return (AbstractRowBuffer<T>)
             new ParquetRowBuffer(
                 onErrorOption,
                 defaultTimezone,
-                allocator,
                 fullyQualifiedChannelName,
                 rowSizeMetric,
                 channelRuntimeState,
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
deleted file mode 100644
index 061405a53..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowFlusher.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright (c) 2022 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.Pair;
-import net.snowflake.ingest.utils.SFException;
-import org.apache.arrow.vector.VectorLoader;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.VectorUnloader;
-import org.apache.arrow.vector.ipc.ArrowStreamWriter;
-import org.apache.arrow.vector.ipc.ArrowWriter;
-import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
-
-/**
- * Converts {@link ChannelData} buffered in {@link RowBuffer} to the Arrow format for faster
- * processing.
- */
-public class ArrowFlusher implements Flusher<VectorSchemaRoot> {
-  private static final Logging logger = new Logging(ArrowFlusher.class);
-
-  @Override
-  public Flusher.SerializationResult serialize(
-      List<ChannelData<VectorSchemaRoot>> channelsDataPerTable, String filePath)
-      throws IOException {
-    ByteArrayOutputStream chunkData = new ByteArrayOutputStream();
-    List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
-    long rowCount = 0L;
-    float chunkUncompressedSize = 0f;
-    VectorSchemaRoot root = null;
-    ArrowWriter arrowWriter = null;
-    VectorLoader loader = null;
-    String firstChannelFullyQualifiedTableName = null;
-    Map<String, RowBufferStats> columnEpStatsMapCombined = null;
-    Pair<Long, Long> chunkMinMaxInsertTimeInMs = null;
-
-    try {
-      for (ChannelData<VectorSchemaRoot> data : channelsDataPerTable) {
-        // Create channel metadata
-        ChannelMetadata channelMetadata =
-            ChannelMetadata.builder()
-                .setOwningChannelFromContext(data.getChannelContext())
-                .setRowSequencer(data.getRowSequencer())
-                .setOffsetToken(data.getOffsetToken())
-                .build();
-        // Add channel metadata to the metadata list
-        channelsMetadataList.add(channelMetadata);
-
-        logger.logDebug(
-            "Start building channel={}, rowCount={}, bufferSize={} in blob={}",
-            data.getChannelContext().getFullyQualifiedName(),
-            data.getRowCount(),
-            data.getBufferSize(),
-            filePath);
-
-        if (root == null) {
-          columnEpStatsMapCombined = data.getColumnEps();
-          root = data.getVectors();
-          arrowWriter = new ArrowStreamWriter(root, null, chunkData);
-          loader = new VectorLoader(root);
-          firstChannelFullyQualifiedTableName =
-              data.getChannelContext().getFullyQualifiedTableName();
-          arrowWriter.start();
-          chunkMinMaxInsertTimeInMs = data.getMinMaxInsertTimeInMs();
-        } else {
-          // This method assumes that channelsDataPerTable is grouped by table. We double check
-          // here and throw an error if the assumption is violated
-          if (!data.getChannelContext()
-              .getFullyQualifiedTableName()
-              .equals(firstChannelFullyQualifiedTableName)) {
-            throw new SFException(ErrorCode.INVALID_DATA_IN_CHUNK);
-          }
-
-          columnEpStatsMapCombined =
-              ChannelData.getCombinedColumnStatsMap(columnEpStatsMapCombined, data.getColumnEps());
-          chunkMinMaxInsertTimeInMs =
-              ChannelData.getCombinedMinMaxInsertTimeInMs(
-                  chunkMinMaxInsertTimeInMs, data.getMinMaxInsertTimeInMs());
-
-          VectorUnloader unloader = new VectorUnloader(data.getVectors());
-          ArrowRecordBatch recordBatch = unloader.getRecordBatch();
-          loader.load(recordBatch);
-          recordBatch.close();
-          data.getVectors().close();
-        }
-
-        // Write channel data using the stream writer
-        arrowWriter.writeBatch();
-        rowCount += data.getRowCount();
-        chunkUncompressedSize += data.getBufferSize();
-
-        logger.logDebug(
-            "Finish building channel={}, rowCount={}, bufferSize={} in blob={}",
-            data.getChannelContext().getFullyQualifiedName(),
-            data.getRowCount(),
-            data.getBufferSize(),
-            filePath);
-      }
-    } finally {
-      if (arrowWriter != null) {
-        arrowWriter.close();
-        root.close();
-      }
-    }
-    return new Flusher.SerializationResult(
-        channelsMetadataList,
-        columnEpStatsMapCombined,
-        rowCount,
-        chunkUncompressedSize,
-        chunkData,
-        chunkMinMaxInsertTimeInMs);
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
deleted file mode 100644
index 3e936233d..000000000
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ArrowRowBuffer.java
+++ /dev/null
@@ -1,813 +0,0 @@
-/*
- * Copyright (c) 2021 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.streaming.internal;
-
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.math.RoundingMode;
-import java.time.ZoneId;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.function.Consumer;
-import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import net.snowflake.ingest.utils.ErrorCode;
-import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.SFException;
-import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.util.VisibleForTesting;
-import org.apache.arrow.vector.BaseFixedWidthVector;
-import org.apache.arrow.vector.BaseVariableWidthVector;
-import org.apache.arrow.vector.BigIntVector;
-import org.apache.arrow.vector.BitVector;
-import org.apache.arrow.vector.DateDayVector;
-import org.apache.arrow.vector.DecimalVector;
-import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.Float8Vector;
-import org.apache.arrow.vector.IntVector;
-import org.apache.arrow.vector.SmallIntVector;
-import org.apache.arrow.vector.TinyIntVector;
-import org.apache.arrow.vector.VarBinaryVector;
-import org.apache.arrow.vector.VarCharVector;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.complex.StructVector;
-import org.apache.arrow.vector.types.Types;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.Field;
-import org.apache.arrow.vector.types.pojo.FieldType;
-import org.apache.arrow.vector.util.Text;
-import org.apache.arrow.vector.util.TransferPair;
-
-/**
- * The buffer in the Streaming Ingest channel that holds the un-flushed rows, these rows will be
- * converted to Arrow format for faster processing
- */
-class ArrowRowBuffer extends AbstractRowBuffer<VectorSchemaRoot> {
-  private static final Logging logger = new Logging(ArrowRowBuffer.class);
-
-  // Constants for column fields
-  private static final String FIELD_EPOCH_IN_SECONDS = "epoch"; // seconds since epoch
-  private static final String FIELD_TIME_ZONE = "timezone"; // time zone index
-  private static final String FIELD_FRACTION_IN_NANOSECONDS = "fraction"; // fraction in nanoseconds
-
-  // Column metadata that will send back to server as part of the blob, and will be used by the
-  // Arrow reader
-  private static final String COLUMN_PHYSICAL_TYPE = "physicalType";
-  private static final String COLUMN_LOGICAL_TYPE = "logicalType";
-  private static final String COLUMN_NULLABLE = "nullable";
-  static final String COLUMN_SCALE = "scale";
-  private static final String COLUMN_PRECISION = "precision";
-  private static final String COLUMN_CHAR_LENGTH = "charLength";
-  private static final String COLUMN_BYTE_LENGTH = "byteLength";
-  @VisibleForTesting static final int DECIMAL_BIT_WIDTH = 128;
-
-  // Holder for a set of the Arrow vectors (buffers)
-  @VisibleForTesting VectorSchemaRoot vectorsRoot;
-
-  // For ABORT on_error option, temp vectors are needed to temporarily holding the rows until
-  // they're all validated, then the rows will be transferred to the final VectorSchemaRoot
-  @VisibleForTesting VectorSchemaRoot tempVectorsRoot;
-
-  // Map the column name to Arrow column field
-  private final Map<String, Field> fields;
-
-  /** Construct a ArrowRowBuffer object. */
-  ArrowRowBuffer(
-      OpenChannelRequest.OnErrorOption onErrorOption,
-      ZoneId defaultTimezone,
-      BufferAllocator allocator,
-      String fullyQualifiedChannelName,
-      Consumer<Float> rowSizeMetric,
-      ChannelRuntimeState channelState) {
-    super(
-        onErrorOption,
-        defaultTimezone,
-        allocator,
-        fullyQualifiedChannelName,
-        rowSizeMetric,
-        channelState);
-    this.fields = new HashMap<>();
-  }
-
-  /**
-   * Setup the column fields and vectors using the column metadata from the server
-   *
-   * @param columns list of column metadata
-   */
-  @Override
-  public void setupSchema(List<ColumnMetadata> columns) {
-    List<FieldVector> vectors = new ArrayList<>();
-    List<FieldVector> tempVectors = new ArrayList<>();
-
-    for (ColumnMetadata column : columns) {
-      validateColumnCollation(column);
-      Field field = buildField(column);
-      FieldVector vector = field.createVector(this.allocator);
-      if (!field.isNullable()) {
-        addNonNullableFieldName(field.getName());
-      }
-      this.fields.put(column.getInternalName(), field);
-      vectors.add(vector);
-      this.statsMap.put(
-          column.getInternalName(), new RowBufferStats(column.getName(), column.getCollation()));
-
-      if (onErrorOption == OpenChannelRequest.OnErrorOption.ABORT) {
-        FieldVector tempVector = field.createVector(this.allocator);
-        tempVectors.add(tempVector);
-        this.tempStatsMap.put(
-            column.getInternalName(), new RowBufferStats(column.getName(), column.getCollation()));
-      }
-    }
-
-    this.vectorsRoot = new VectorSchemaRoot(vectors);
-    this.tempVectorsRoot = new VectorSchemaRoot(tempVectors);
-  }
-
-  /** Close the row buffer by releasing its internal resources. */
-  @Override
-  public void closeInternal() {
-    if (this.vectorsRoot != null) {
-      this.vectorsRoot.close();
-      this.tempVectorsRoot.close();
-    }
-    this.fields.clear();
-  }
-
-  /** Reset the variables after each flush. Note that the caller needs to handle synchronization */
-  @Override
-  void reset() {
-    super.reset();
-    this.vectorsRoot.clear();
-  }
-
-  /**
-   * Build the column field from the column metadata
-   *
-   * @param column column metadata
-   * @return Column field object
-   */
-  Field buildField(ColumnMetadata column) {
-    ArrowType arrowType;
-    FieldType fieldType;
-    List<Field> children = null;
-
-    // Put info into the metadata, which will be used by the Arrow reader later
-    Map<String, String> metadata = new HashMap<>();
-    metadata.put(COLUMN_LOGICAL_TYPE, column.getLogicalType());
-    metadata.put(COLUMN_PHYSICAL_TYPE, column.getPhysicalType());
-    metadata.put(COLUMN_NULLABLE, String.valueOf(column.getNullable()));
-
-    ColumnPhysicalType physicalType;
-    ColumnLogicalType logicalType;
-    try {
-      physicalType = ColumnPhysicalType.valueOf(column.getPhysicalType());
-      logicalType = ColumnLogicalType.valueOf(column.getLogicalType());
-    } catch (IllegalArgumentException e) {
-      throw new SFException(
-          ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
-    }
-
-    if (column.getPrecision() != null) {
-      metadata.put(COLUMN_PRECISION, column.getPrecision().toString());
-    }
-    if (column.getScale() != null) {
-      metadata.put(COLUMN_SCALE, column.getScale().toString());
-    }
-    if (column.getByteLength() != null) {
-      metadata.put(COLUMN_BYTE_LENGTH, column.getByteLength().toString());
-    }
-    if (column.getLength() != null) {
-      metadata.put(COLUMN_CHAR_LENGTH, column.getLength().toString());
-    }
-
-    // Handle differently depends on the column logical and physical types
-    switch (logicalType) {
-      case FIXED:
-        if ((column.getScale() != null && column.getScale() != 0)
-            || physicalType == ColumnPhysicalType.SB16) {
-          arrowType =
-              new ArrowType.Decimal(column.getPrecision(), column.getScale(), DECIMAL_BIT_WIDTH);
-        } else {
-          switch (physicalType) {
-            case SB1:
-              arrowType = Types.MinorType.TINYINT.getType();
-              break;
-            case SB2:
-              arrowType = Types.MinorType.SMALLINT.getType();
-              break;
-            case SB4:
-              arrowType = Types.MinorType.INT.getType();
-              break;
-            case SB8:
-              arrowType = Types.MinorType.BIGINT.getType();
-              break;
-            default:
-              throw new SFException(
-                  ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
-          }
-        }
-        break;
-      case ANY:
-      case ARRAY:
-      case CHAR:
-      case TEXT:
-      case OBJECT:
-      case VARIANT:
-        arrowType = Types.MinorType.VARCHAR.getType();
-        break;
-      case TIMESTAMP_LTZ:
-      case TIMESTAMP_NTZ:
-        switch (physicalType) {
-          case SB8:
-            arrowType = Types.MinorType.BIGINT.getType();
-            break;
-          case SB16:
-            {
-              arrowType = Types.MinorType.STRUCT.getType();
-              FieldType fieldTypeEpoch =
-                  new FieldType(true, Types.MinorType.BIGINT.getType(), null, metadata);
-              FieldType fieldTypeFraction =
-                  new FieldType(true, Types.MinorType.INT.getType(), null, metadata);
-              Field fieldEpoch = new Field(FIELD_EPOCH_IN_SECONDS, fieldTypeEpoch, null);
-              Field fieldFraction =
-                  new Field(FIELD_FRACTION_IN_NANOSECONDS, fieldTypeFraction, null);
-              children = new ArrayList<>();
-              children.add(fieldEpoch);
-              children.add(fieldFraction);
-              break;
-            }
-          default:
-            throw new SFException(
-                ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
-        }
-        break;
-      case TIMESTAMP_TZ:
-        switch (physicalType) {
-          case SB8:
-            {
-              arrowType = Types.MinorType.STRUCT.getType();
-              FieldType fieldTypeEpoch =
-                  new FieldType(true, Types.MinorType.BIGINT.getType(), null, metadata);
-              FieldType fieldTypeTimezone =
-                  new FieldType(true, Types.MinorType.INT.getType(), null, metadata);
-              Field fieldEpoch = new Field(FIELD_EPOCH_IN_SECONDS, fieldTypeEpoch, null);
-              Field fieldTimezone = new Field(FIELD_TIME_ZONE, fieldTypeTimezone, null);
-
-              children = new ArrayList<>();
-              children.add(fieldEpoch);
-              children.add(fieldTimezone);
-              break;
-            }
-          case SB16:
-            {
-              arrowType = Types.MinorType.STRUCT.getType();
-              FieldType fieldTypeEpoch =
-                  new FieldType(true, Types.MinorType.BIGINT.getType(), null, metadata);
-              FieldType fieldTypeFraction =
-                  new FieldType(true, Types.MinorType.INT.getType(), null, metadata);
-              FieldType fieldTypeTimezone =
-                  new FieldType(true, Types.MinorType.INT.getType(), null, metadata);
-              Field fieldEpoch = new Field(FIELD_EPOCH_IN_SECONDS, fieldTypeEpoch, null);
-              Field fieldFraction =
-                  new Field(FIELD_FRACTION_IN_NANOSECONDS, fieldTypeFraction, null);
-              Field fieldTimezone = new Field(FIELD_TIME_ZONE, fieldTypeTimezone, null);
-
-              children = new ArrayList<>();
-              children.add(fieldEpoch);
-              children.add(fieldFraction);
-              children.add(fieldTimezone);
-              break;
-            }
-          default:
-            throw new SFException(
-                ErrorCode.UNKNOWN_DATA_TYPE,
-                "Unknown physical type for TIMESTAMP_TZ: " + physicalType);
-        }
-        break;
-      case DATE:
-        arrowType = Types.MinorType.DATEDAY.getType();
-        break;
-      case TIME:
-        switch (physicalType) {
-          case SB4:
-            arrowType = Types.MinorType.INT.getType();
-            break;
-          case SB8:
-            arrowType = Types.MinorType.BIGINT.getType();
-            break;
-          default:
-            throw new SFException(
-                ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
-        }
-        break;
-      case BOOLEAN:
-        arrowType = Types.MinorType.BIT.getType();
-        break;
-      case BINARY:
-        arrowType = Types.MinorType.VARBINARY.getType();
-        break;
-      case REAL:
-        arrowType = Types.MinorType.FLOAT8.getType();
-        break;
-      default:
-        throw new SFException(
-            ErrorCode.UNKNOWN_DATA_TYPE, column.getLogicalType(), column.getPhysicalType());
-    }
-
-    // Create the corresponding column field base on the column data type
-    fieldType = new FieldType(column.getNullable(), arrowType, null, metadata);
-    return new Field(column.getInternalName(), fieldType, children);
-  }
-
-  @Override
-  void moveTempRowsToActualBuffer(int tempRowCount) {
-    // If all the rows are inserted successfully, transfer the rows from temp vectors to
-    // the final vectors and update the row size and row count
-    // TODO: switch to VectorSchemaRootAppender once it works for all vector types
-    for (Field field : fields.values()) {
-      FieldVector from = this.tempVectorsRoot.getVector(field);
-      FieldVector to = this.vectorsRoot.getVector(field);
-      for (int rowIdx = 0; rowIdx < tempRowCount; rowIdx++) {
-        to.copyFromSafe(rowIdx, this.bufferedRowCount + rowIdx, from);
-      }
-    }
-  }
-
-  @Override
-  void clearTempRows() {
-    tempVectorsRoot.clear();
-  }
-
-  @Override
-  boolean hasColumns() {
-    return !fields.isEmpty();
-  }
-
-  @Override
-  Optional<VectorSchemaRoot> getSnapshot(final String filePath) {
-    List<FieldVector> oldVectors = new ArrayList<>();
-    for (FieldVector vector : this.vectorsRoot.getFieldVectors()) {
-      vector.setValueCount(this.bufferedRowCount);
-      if (vector instanceof DecimalVector) {
-        // DecimalVectors do not transfer FieldType metadata when using
-        // vector.getTransferPair. We need to explicitly create the new vector to transfer to
-        // in order to keep the metadata.
-        ArrowType arrowType =
-            new ArrowType.Decimal(
-                ((DecimalVector) vector).getPrecision(),
-                ((DecimalVector) vector).getScale(),
-                DECIMAL_BIT_WIDTH);
-        FieldType fieldType =
-            new FieldType(
-                vector.getField().isNullable(), arrowType, null, vector.getField().getMetadata());
-        Field f = new Field(vector.getName(), fieldType, null);
-        DecimalVector newVector = new DecimalVector(f, this.allocator);
-        TransferPair t = vector.makeTransferPair(newVector);
-        t.transfer();
-        oldVectors.add((FieldVector) t.getTo());
-      } else {
-        TransferPair t = vector.getTransferPair(this.allocator);
-        t.transfer();
-        oldVectors.add((FieldVector) t.getTo());
-      }
-    }
-    VectorSchemaRoot root = new VectorSchemaRoot(oldVectors);
-    root.setRowCount(this.bufferedRowCount);
-    return oldVectors.isEmpty() ? Optional.empty() : Optional.of(root);
-  }
-
-  @Override
-  boolean hasColumn(String name) {
-    return this.fields.get(name) != null;
-  }
-
-  @Override
-  float addRow(
-      Map<String, Object> row,
-      int bufferedRowIndex,
-      Map<String, RowBufferStats> statsMap,
-      Set<String> formattedInputColumnNames,
-      final long insertRowIndex) {
-    return convertRowToArrow(
-        row, vectorsRoot, bufferedRowIndex, statsMap, formattedInputColumnNames, insertRowIndex);
-  }
-
-  @Override
-  float addTempRow(
-      Map<String, Object> row,
-      int curRowIndex,
-      Map<String, RowBufferStats> statsMap,
-      Set<String> formattedInputColumnNames,
-      long insertRowIndex) {
-    return convertRowToArrow(
-        row, tempVectorsRoot, curRowIndex, statsMap, formattedInputColumnNames, insertRowIndex);
-  }
-
-  /**
-   * Convert the input row to the correct Arrow format
-   *
-   * @param row input row
-   * @param sourceVectors vectors (buffers) that hold the row
-   * @param bufferedRowIndex Buffered row index. This is not the same as the input row index
-   * @param statsMap column stats map
-   * @param inputColumnNames list of input column names after formatting
-   * @param insertRowsCurrIndex Row index of the input Rows passed in {@link
-   *     net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel#insertRows(Iterable,
-   *     String)}
-   * @return row size
-   */
-  private float convertRowToArrow(
-      Map<String, Object> row,
-      VectorSchemaRoot sourceVectors,
-      int bufferedRowIndex,
-      Map<String, RowBufferStats> statsMap,
-      Set<String> inputColumnNames,
-      long insertRowsCurrIndex) {
-    // Insert values to the corresponding arrow buffers
-    float rowBufferSize = 0F;
-    // Create new empty stats just for the current row.
-    Map<String, RowBufferStats> forkedStatsMap = new HashMap<>();
-
-    for (Map.Entry<String, Object> entry : row.entrySet()) {
-      rowBufferSize += 0.125; // 1/8 for null value bitmap
-      String columnName = LiteralQuoteUtils.unquoteColumnName(entry.getKey());
-      Object value = entry.getValue();
-      Field field = this.fields.get(columnName);
-      Utils.assertNotNull("Arrow column field", field);
-      FieldVector vector = sourceVectors.getVector(field);
-      Utils.assertNotNull("Arrow column vector", vector);
-      RowBufferStats forkedStats = statsMap.get(columnName).forkEmpty();
-      forkedStatsMap.put(columnName, forkedStats);
-      Utils.assertNotNull("Arrow column stats", forkedStats);
-      ColumnLogicalType logicalType =
-          ColumnLogicalType.valueOf(field.getMetadata().get(COLUMN_LOGICAL_TYPE));
-      ColumnPhysicalType physicalType =
-          ColumnPhysicalType.valueOf(field.getMetadata().get(COLUMN_PHYSICAL_TYPE));
-
-      boolean isParsedValueNull = false;
-      if (value != null) {
-        switch (logicalType) {
-          case FIXED:
-            int columnPrecision = Integer.parseInt(field.getMetadata().get(COLUMN_PRECISION));
-            int columnScale = getColumnScale(field.getMetadata());
-            BigDecimal inputAsBigDecimal =
-                DataValidationUtil.validateAndParseBigDecimal(
-                    forkedStats.getColumnDisplayName(), value, insertRowsCurrIndex);
-            // vector.setSafe requires the BigDecimal input scale explicitly match its scale
-            inputAsBigDecimal = inputAsBigDecimal.setScale(columnScale, RoundingMode.HALF_UP);
-
-            DataValidationUtil.checkValueInRange(
-                inputAsBigDecimal, columnScale, columnPrecision, insertRowsCurrIndex);
-
-            if (columnScale != 0 || physicalType == ColumnPhysicalType.SB16) {
-              ((DecimalVector) vector).setSafe(bufferedRowIndex, inputAsBigDecimal);
-              forkedStats.addIntValue(inputAsBigDecimal.unscaledValue());
-              rowBufferSize += 16;
-            } else {
-              switch (physicalType) {
-                case SB1:
-                  ((TinyIntVector) vector)
-                      .setSafe(bufferedRowIndex, inputAsBigDecimal.byteValueExact());
-                  forkedStats.addIntValue(inputAsBigDecimal.toBigInteger());
-                  rowBufferSize += 1;
-                  break;
-                case SB2:
-                  ((SmallIntVector) vector)
-                      .setSafe(bufferedRowIndex, inputAsBigDecimal.shortValueExact());
-                  forkedStats.addIntValue(inputAsBigDecimal.toBigInteger());
-                  rowBufferSize += 2;
-                  break;
-                case SB4:
-                  ((IntVector) vector).setSafe(bufferedRowIndex, inputAsBigDecimal.intValueExact());
-                  forkedStats.addIntValue(inputAsBigDecimal.toBigInteger());
-                  rowBufferSize += 4;
-                  break;
-                case SB8:
-                  ((BigIntVector) vector)
-                      .setSafe(bufferedRowIndex, inputAsBigDecimal.longValueExact());
-                  forkedStats.addIntValue(inputAsBigDecimal.toBigInteger());
-                  rowBufferSize += 8;
-                  break;
-                default:
-                  throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
-              }
-            }
-            break;
-          case ANY:
-          case CHAR:
-          case TEXT:
-            {
-              String maxLengthString = field.getMetadata().get(COLUMN_CHAR_LENGTH);
-              String str =
-                  DataValidationUtil.validateAndParseString(
-                      forkedStats.getColumnDisplayName(),
-                      value,
-                      Optional.ofNullable(maxLengthString).map(Integer::parseInt),
-                      insertRowsCurrIndex);
-              Text text = new Text(str);
-              ((VarCharVector) vector).setSafe(bufferedRowIndex, text);
-              forkedStats.addStrValue(str);
-              rowBufferSize += text.getBytes().length;
-              break;
-            }
-          case OBJECT:
-            {
-              String str =
-                  DataValidationUtil.validateAndParseObject(
-                      forkedStats.getColumnDisplayName(), value, insertRowsCurrIndex);
-              Text text = new Text(str);
-              ((VarCharVector) vector).setSafe(bufferedRowIndex, text);
-              rowBufferSize += text.getBytes().length;
-              break;
-            }
-          case ARRAY:
-            {
-              String str =
-                  DataValidationUtil.validateAndParseArray(
-                      forkedStats.getColumnDisplayName(), value, insertRowsCurrIndex);
-              Text text = new Text(str);
-              ((VarCharVector) vector).setSafe(bufferedRowIndex, text);
-              rowBufferSize += text.getBytes().length;
-              break;
-            }
-          case VARIANT:
-            {
-              String str =
-                  DataValidationUtil.validateAndParseVariant(
-                      forkedStats.getColumnDisplayName(), value, insertRowsCurrIndex);
-              if (str != null) {
-                Text text = new Text(str);
-                ((VarCharVector) vector).setSafe(bufferedRowIndex, text);
-                rowBufferSize += text.getBytes().length;
-              } else {
-                isParsedValueNull = true;
-              }
-              break;
-            }
-          case TIMESTAMP_LTZ:
-          case TIMESTAMP_NTZ:
-            boolean trimTimezone = logicalType == ColumnLogicalType.TIMESTAMP_NTZ;
-
-            switch (physicalType) {
-              case SB8:
-                {
-                  BigIntVector bigIntVector = (BigIntVector) vector;
-                  TimestampWrapper timestampWrapper =
-                      DataValidationUtil.validateAndParseTimestamp(
-                          forkedStats.getColumnDisplayName(),
-                          value,
-                          getColumnScale(field.getMetadata()),
-                          defaultTimezone,
-                          trimTimezone,
-                          insertRowsCurrIndex);
-                  BigInteger timestampBinary = timestampWrapper.toBinary(false);
-                  bigIntVector.setSafe(bufferedRowIndex, timestampBinary.longValue());
-                  forkedStats.addIntValue(timestampBinary);
-                  rowBufferSize += 8;
-                  break;
-                }
-              case SB16:
-                {
-                  StructVector structVector = (StructVector) vector;
-                  BigIntVector epochVector =
-                      (BigIntVector) structVector.getChild(FIELD_EPOCH_IN_SECONDS);
-                  IntVector fractionVector =
-                      (IntVector) structVector.getChild(FIELD_FRACTION_IN_NANOSECONDS);
-                  rowBufferSize += 0.25; // for children vector's null value
-                  structVector.setIndexDefined(bufferedRowIndex);
-
-                  TimestampWrapper timestampWrapper =
-                      DataValidationUtil.validateAndParseTimestamp(
-                          forkedStats.getColumnDisplayName(),
-                          value,
-                          getColumnScale(field.getMetadata()),
-                          defaultTimezone,
-                          trimTimezone,
-                          insertRowsCurrIndex);
-                  epochVector.setSafe(bufferedRowIndex, timestampWrapper.getEpoch());
-                  fractionVector.setSafe(bufferedRowIndex, timestampWrapper.getFraction());
-                  rowBufferSize += 12;
-                  forkedStats.addIntValue(timestampWrapper.toBinary(false));
-                  break;
-                }
-              default:
-                throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
-            }
-            break;
-          case TIMESTAMP_TZ:
-            switch (physicalType) {
-              case SB8:
-                {
-                  StructVector structVector = (StructVector) vector;
-                  BigIntVector epochVector =
-                      (BigIntVector) structVector.getChild(FIELD_EPOCH_IN_SECONDS);
-                  IntVector timezoneVector = (IntVector) structVector.getChild(FIELD_TIME_ZONE);
-
-                  rowBufferSize += 0.25; // for children vector's null value
-                  structVector.setIndexDefined(bufferedRowIndex);
-
-                  TimestampWrapper timestampWrapper =
-                      DataValidationUtil.validateAndParseTimestamp(
-                          forkedStats.getColumnDisplayName(),
-                          value,
-                          getColumnScale(field.getMetadata()),
-                          defaultTimezone,
-                          false,
-                          insertRowsCurrIndex);
-                  epochVector.setSafe(
-                      bufferedRowIndex, timestampWrapper.toBinary(false).longValueExact());
-                  timezoneVector.setSafe(bufferedRowIndex, timestampWrapper.getTimeZoneIndex());
-                  rowBufferSize += 12;
-                  forkedStats.addIntValue(timestampWrapper.toBinary(true));
-                  break;
-                }
-              case SB16:
-                {
-                  StructVector structVector = (StructVector) vector;
-                  BigIntVector epochVector =
-                      (BigIntVector) structVector.getChild(FIELD_EPOCH_IN_SECONDS);
-                  IntVector fractionVector =
-                      (IntVector) structVector.getChild(FIELD_FRACTION_IN_NANOSECONDS);
-                  IntVector timezoneVector = (IntVector) structVector.getChild(FIELD_TIME_ZONE);
-
-                  rowBufferSize += 0.375; // for children vector's null value
-                  structVector.setIndexDefined(bufferedRowIndex);
-
-                  TimestampWrapper timestampWrapper =
-                      DataValidationUtil.validateAndParseTimestamp(
-                          forkedStats.getColumnDisplayName(),
-                          value,
-                          getColumnScale(field.getMetadata()),
-                          defaultTimezone,
-                          false,
-                          insertRowsCurrIndex);
-                  epochVector.setSafe(bufferedRowIndex, timestampWrapper.getEpoch());
-                  fractionVector.setSafe(bufferedRowIndex, timestampWrapper.getFraction());
-                  timezoneVector.setSafe(bufferedRowIndex, timestampWrapper.getTimeZoneIndex());
-                  rowBufferSize += 16;
-                  BigInteger timeInBinary = timestampWrapper.toBinary(true);
-                  forkedStats.addIntValue(timeInBinary);
-                  break;
-                }
-              default:
-                throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
-            }
-            break;
-          case DATE:
-            {
-              DateDayVector dateDayVector = (DateDayVector) vector;
-              // Expect days past the epoch
-              int intValue =
-                  DataValidationUtil.validateAndParseDate(
-                      forkedStats.getColumnDisplayName(), value, insertRowsCurrIndex);
-              dateDayVector.setSafe(bufferedRowIndex, intValue);
-              forkedStats.addIntValue(BigInteger.valueOf(intValue));
-              rowBufferSize += 4;
-              break;
-            }
-          case TIME:
-            switch (physicalType) {
-              case SB4:
-                {
-                  BigInteger timeInScale =
-                      DataValidationUtil.validateAndParseTime(
-                          forkedStats.getColumnDisplayName(),
-                          value,
-                          getColumnScale(field.getMetadata()),
-                          insertRowsCurrIndex);
-                  ((IntVector) vector).setSafe(bufferedRowIndex, timeInScale.intValue());
-                  forkedStats.addIntValue(timeInScale);
-                  rowBufferSize += 4;
-                  break;
-                }
-              case SB8:
-                {
-                  BigInteger timeInScale =
-                      DataValidationUtil.validateAndParseTime(
-                          forkedStats.getColumnDisplayName(),
-                          value,
-                          getColumnScale(field.getMetadata()),
-                          insertRowsCurrIndex);
-                  ((BigIntVector) vector).setSafe(bufferedRowIndex, timeInScale.longValue());
-                  forkedStats.addIntValue(timeInScale);
-                  rowBufferSize += 8;
-                  break;
-                }
-              default:
-                throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
-            }
-            break;
-          case BOOLEAN:
-            {
-              int intValue =
-                  DataValidationUtil.validateAndParseBoolean(
-                      forkedStats.getColumnDisplayName(), value, insertRowsCurrIndex);
-              ((BitVector) vector).setSafe(bufferedRowIndex, intValue);
-              rowBufferSize += 0.125;
-              forkedStats.addIntValue(BigInteger.valueOf(intValue));
-              break;
-            }
-          case BINARY:
-            String maxLengthString = field.getMetadata().get(COLUMN_BYTE_LENGTH);
-            byte[] bytes =
-                DataValidationUtil.validateAndParseBinary(
-                    forkedStats.getColumnDisplayName(),
-                    value,
-                    Optional.ofNullable(maxLengthString).map(Integer::parseInt),
-                    insertRowsCurrIndex);
-            ((VarBinaryVector) vector).setSafe(bufferedRowIndex, bytes);
-            forkedStats.addBinaryValue(bytes);
-            rowBufferSize += bytes.length;
-            break;
-          case REAL:
-            double doubleValue =
-                DataValidationUtil.validateAndParseReal(
-                    forkedStats.getColumnDisplayName(), value, insertRowsCurrIndex);
-            ((Float8Vector) vector).setSafe(bufferedRowIndex, doubleValue);
-            forkedStats.addRealValue(doubleValue);
-            rowBufferSize += 8;
-            break;
-          default:
-            throw new SFException(ErrorCode.UNKNOWN_DATA_TYPE, logicalType, physicalType);
-        }
-      }
-
-      if (value == null || isParsedValueNull) {
-        if (!field.getFieldType().isNullable()) {
-          throw new SFException(
-              ErrorCode.INVALID_FORMAT_ROW, columnName, "Passed null to non nullable field");
-        } else {
-          insertNull(vector, forkedStats, bufferedRowIndex);
-        }
-      }
-    }
-
-    // All input values passed validation, iterate over the columns again and combine their existing
-    // statistics with the forked statistics for the current row.
-    for (Map.Entry<String, RowBufferStats> forkedColStats : forkedStatsMap.entrySet()) {
-      String columnName = forkedColStats.getKey();
-      statsMap.put(
-          columnName,
-          RowBufferStats.getCombinedStats(statsMap.get(columnName), forkedColStats.getValue()));
-    }
-
-    // Insert nulls to the columns that doesn't show up in the input
-    for (String columnName : Sets.difference(this.fields.keySet(), inputColumnNames)) {
-      rowBufferSize += 0.125; // 1/8 for null value bitmap
-      insertNull(
-          sourceVectors.getVector(this.fields.get(columnName)),
-          statsMap.get(columnName),
-          bufferedRowIndex);
-    }
-
-    return rowBufferSize;
-  }
-
-  /** Helper function to insert null value to a field vector */
-  private void insertNull(FieldVector vector, RowBufferStats stats, int curRowIndex) {
-    if (BaseFixedWidthVector.class.isAssignableFrom(vector.getClass())) {
-      ((BaseFixedWidthVector) vector).setNull(curRowIndex);
-    } else if (BaseVariableWidthVector.class.isAssignableFrom(vector.getClass())) {
-      ((BaseVariableWidthVector) vector).setNull(curRowIndex);
-    } else if (vector instanceof StructVector) {
-      ((StructVector) vector).setNull(curRowIndex);
-      ((StructVector) vector)
-          .getChildrenFromFields()
-          .forEach(
-              child -> {
-                ((BaseFixedWidthVector) child).setNull(curRowIndex);
-              });
-    } else {
-      throw new SFException(ErrorCode.INTERNAL_ERROR, "Unexpected FieldType");
-    }
-    stats.incCurrentNullCount();
-  }
-
-  private int getColumnScale(Map<String, String> metadata) {
-    return Integer.parseInt(metadata.get(ArrowRowBuffer.COLUMN_SCALE));
-  }
-
-  @Override
-  public Flusher<VectorSchemaRoot> createFlusher() {
-    return new ArrowFlusher();
-  }
-
-  @VisibleForTesting
-  @Override
-  Object getVectorValueAt(String column, int index) {
-    Object value = vectorsRoot.getVector(column).getObject(index);
-    return (value instanceof Text) ? new String(((Text) value).getBytes()) : value;
-  }
-
-  @VisibleForTesting
-  int getTempRowCount() {
-    return tempVectorsRoot.getRowCount();
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
index 7d7c139c1..9442b2774 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
@@ -11,7 +11,6 @@
 import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
 import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
 import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
-import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
 import static net.snowflake.ingest.utils.Utils.toByteArray;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -25,7 +24,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.zip.CRC32;
-import java.util.zip.GZIPOutputStream;
 import javax.crypto.BadPaddingException;
 import javax.crypto.IllegalBlockSizeException;
 import javax.crypto.NoSuchPaddingException;
@@ -48,8 +46,8 @@
  *   <li>variable size chunks metadata in json format
  * </ul>
  *
- * <p>After the metadata, it will be one or more chunks of variable size Arrow/Parquet data, and
- * each chunk will be encrypted and compressed separately.
+ * <p>After the metadata, it will be one or more chunks of variable size Parquet data, and each
+ * chunk will be encrypted and compressed separately.
  */
 class BlobBuilder {
 
@@ -86,14 +84,10 @@ static <T> Blob constructBlobAndMetadata(
 
       if (!serializedChunk.channelsMetadataList.isEmpty()) {
         ByteArrayOutputStream chunkData = serializedChunk.chunkData;
-        Pair<byte[], Integer> compressionResult =
-            compressIfNeededAndPadChunk(
-                filePath,
-                chunkData,
-                Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES,
-                bdecVersion == Constants.BdecVersion.ONE);
-        byte[] compressedAndPaddedChunkData = compressionResult.getFirst();
-        int compressedChunkLength = compressionResult.getSecond();
+        Pair<byte[], Integer> paddedChunk =
+            padChunk(chunkData, Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES);
+        byte[] paddedChunkData = paddedChunk.getFirst();
+        int paddedChunkLength = paddedChunk.getSecond();
 
         // Encrypt the compressed chunk data, the encryption key is derived using the key from
         // server with the full blob path.
@@ -103,13 +97,10 @@ static <T> Blob constructBlobAndMetadata(
         long iv = curDataSize / Constants.ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES;
         byte[] encryptedCompressedChunkData =
             Cryptor.encrypt(
-                compressedAndPaddedChunkData,
-                firstChannelFlushContext.getEncryptionKey(),
-                filePath,
-                iv);
+                paddedChunkData, firstChannelFlushContext.getEncryptionKey(), filePath, iv);
 
         // Compute the md5 of the chunk data
-        String md5 = computeMD5(encryptedCompressedChunkData, compressedChunkLength);
+        String md5 = computeMD5(encryptedCompressedChunkData, paddedChunkLength);
         int encryptedCompressedChunkDataSize = encryptedCompressedChunkData.length;
 
         // Create chunk metadata
@@ -120,9 +111,9 @@ static <T> Blob constructBlobAndMetadata(
                 // The start offset will be updated later in BlobBuilder#build to include the blob
                 // header
                 .setChunkStartOffset(startOffset)
-                // The compressedChunkLength is used because it is the actual data size used for
+                // The paddedChunkLength is used because it is the actual data size used for
                 // decompression and md5 calculation on server side.
-                .setChunkLength(compressedChunkLength)
+                .setChunkLength(paddedChunkLength)
                 .setChannelList(serializedChunk.channelsMetadataList)
                 .setChunkMD5(md5)
                 .setEncryptionKeyId(firstChannelFlushContext.getEncryptionKeyId())
@@ -141,14 +132,14 @@ static <T> Blob constructBlobAndMetadata(
 
         logger.logInfo(
             "Finish building chunk in blob={}, table={}, rowCount={}, startOffset={},"
-                + " uncompressedSize={}, compressedChunkLength={}, encryptedCompressedSize={},"
+                + " uncompressedSize={}, paddedChunkLength={}, encryptedCompressedSize={},"
                 + " bdecVersion={}",
             filePath,
             firstChannelFlushContext.getFullyQualifiedTableName(),
             serializedChunk.rowCount,
             startOffset,
             serializedChunk.chunkUncompressedSize,
-            compressedChunkLength,
+            paddedChunkLength,
             encryptedCompressedChunkDataSize,
             bdecVersion);
       }
@@ -161,84 +152,22 @@ static <T> Blob constructBlobAndMetadata(
   }
 
   /**
-   * Gzip compress the given chunk data
+   * Pad the compressed data for encryption. Encryption needs padding to the
+   * ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES to align with decryption on the Snowflake query path
+   * starting from this chunk offset.
    *
-   * @param filePath blob file full path
    * @param chunkData uncompressed chunk data
    * @param blockSizeToAlignTo block size to align to for encryption
    * @return padded compressed chunk data, aligned to blockSizeToAlignTo, and actual length of
    *     compressed data before padding at the end
    * @throws IOException
    */
-  static Pair<byte[], Integer> compress(
-      String filePath, ByteArrayOutputStream chunkData, int blockSizeToAlignTo) throws IOException {
-    int uncompressedSize = chunkData.size();
-    ByteArrayOutputStream compressedOutputStream = new ByteArrayOutputStream(uncompressedSize);
-    try (GZIPOutputStream gzipOutputStream = new GZIPOutputStream(compressedOutputStream, true)) {
-      gzipOutputStream.write(chunkData.toByteArray());
-    }
-    int firstCompressedSize = compressedOutputStream.size();
-
-    // Based on current experiment, compressing twice will give us the best compression
-    // ratio and compression time combination
-    int doubleCompressedSize = 0;
-    if (COMPRESS_BLOB_TWICE) {
-      ByteArrayOutputStream doubleCompressedOutputStream =
-          new ByteArrayOutputStream(firstCompressedSize);
-      try (GZIPOutputStream doubleGzipOutputStream =
-          new GZIPOutputStream(doubleCompressedOutputStream, true)) {
-        doubleGzipOutputStream.write(compressedOutputStream.toByteArray());
-      }
-      doubleCompressedSize = doubleCompressedOutputStream.size();
-      compressedOutputStream = doubleCompressedOutputStream;
-    }
-
-    logger.logDebug(
-        "Finish compressing chunk in blob={}, uncompressedSize={}, firstCompressedSize={},"
-            + " doubleCompressedSize={}",
-        filePath,
-        uncompressedSize,
-        firstCompressedSize,
-        doubleCompressedSize);
-
-    int compressedSize = compressedOutputStream.size();
-    int paddingSize = blockSizeToAlignTo - compressedSize % blockSizeToAlignTo;
-    compressedOutputStream.write(new byte[paddingSize]);
-    return new Pair<>(compressedOutputStream.toByteArray(), compressedSize);
-  }
-
-  /**
-   * Gzip compress the given chunk data if required by the given write mode and pads the compressed
-   * data for encryption. Only for Arrow. For Parquet the compression is done in the Parquet
-   * library.
-   *
-   * @param filePath blob file full path
-   * @param chunkData uncompressed chunk data
-   * @param blockSizeToAlignTo block size to align to for encryption
-   * @param compress whether to compress the chunk
-   * @return padded compressed chunk data, aligned to blockSizeToAlignTo, and actual length of
-   *     compressed data before padding at the end
-   * @throws IOException
-   */
-  static Pair<byte[], Integer> compressIfNeededAndPadChunk(
-      String filePath, ByteArrayOutputStream chunkData, int blockSizeToAlignTo, boolean compress)
+  static Pair<byte[], Integer> padChunk(ByteArrayOutputStream chunkData, int blockSizeToAlignTo)
       throws IOException {
-    // Encryption needs padding to the ENCRYPTION_ALGORITHM_BLOCK_SIZE_BYTES
-    // to align with decryption on the Snowflake query path starting from this chunk offset.
-    // The padding does not have arrow data and not compressed.
-    // Hence, the actual chunk size is smaller by the padding size.
-    // The compression on the Snowflake query path needs the correct size of the compressed
-    // data.
-    if (compress) {
-      // Stream write mode does not support column level compression.
-      // Compress the chunk data and pad it for encryption.
-      return BlobBuilder.compress(filePath, chunkData, blockSizeToAlignTo);
-    } else {
-      int actualSize = chunkData.size();
-      int paddingSize = blockSizeToAlignTo - actualSize % blockSizeToAlignTo;
-      chunkData.write(new byte[paddingSize]);
-      return new Pair<>(chunkData.toByteArray(), actualSize);
-    }
+    int actualSize = chunkData.size();
+    int paddingSize = blockSizeToAlignTo - actualSize % blockSizeToAlignTo;
+    chunkData.write(new byte[paddingSize]);
+    return new Pair<>(chunkData.toByteArray(), actualSize);
   }
 
   /**
@@ -285,7 +214,7 @@ static byte[] buildBlob(
       blob.write(chunkData);
     }
 
-    // We need to update the start offset for the EP and Arrow/Parquet data in the request since
+    // We need to update the start offset for the EP and Parquet data in the request since
     // some metadata was added at the beginning
     for (ChunkMetadata chunkMetadata : chunksMetadataList) {
       chunkMetadata.advanceStartOffset(metadataSize);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index 392bd53fd..d2cf31a6c 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -6,10 +6,10 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.annotations.VisibleForTesting;
 import java.util.List;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ParameterProvider;
-import org.apache.arrow.util.VisibleForTesting;
 
 /** Metadata for a blob that sends to Snowflake as part of the register blob request */
 class BlobMetadata {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
index 2cfeefc00..3926832fb 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelCache.java
@@ -14,8 +14,7 @@
  * during flush. The key is a fully qualified table name and the value is a set of channels that
  * belongs to this table
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data ({@link ParquetChunkData})
  */
 class ChannelCache<T> {
   // Cache to hold all the valid channels, the key for the outer map is FullyQualifiedTableName and
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
index 07ce4b989..cd4dabaa6 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
@@ -15,8 +15,7 @@
  * Contains the data and metadata returned for each channel flush, which will be used to build the
  * blob and register blob request
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or Parquet
- *     {@link ParquetChunkData}
+ * @param <T> type of column data (Parquet {@link ParquetChunkData}
  */
 class ChannelData<T> {
   private Long rowSequencer;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 7f38c1324..ae4d04a23 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -12,6 +12,7 @@
 import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
 import com.codahale.metrics.Timer;
+import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.security.InvalidAlgorithmParameterException;
@@ -46,8 +47,6 @@
 import net.snowflake.ingest.utils.Pair;
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.util.VisibleForTesting;
-import org.apache.arrow.vector.VectorSchemaRoot;
 
 /**
  * Responsible for flushing data from client to Snowflake tables. When a flush is triggered, it will
@@ -57,8 +56,7 @@
  * <li>upload the blob to stage
  * <li>register the blob to the targeted Snowflake table
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data ({@link ParquetChunkData})
  */
 class FlushService<T> {
 
@@ -626,9 +624,6 @@ <CD> void invalidateAllChannelsInBlob(List<List<ChannelData<CD>>> blobData) {
         chunkData ->
             chunkData.forEach(
                 channelData -> {
-                  if (channelData.getVectors() instanceof VectorSchemaRoot) {
-                    ((VectorSchemaRoot) channelData.getVectors()).close();
-                  }
                   this.owningClient
                       .getChannelCache()
                       .invalidateChannelIfSequencersMatch(
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
index fc9d1c858..e5a562f01 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
@@ -14,8 +14,7 @@
  * Interface to convert {@link ChannelData} buffered in {@link RowBuffer} to the underlying format
  * implementation for faster processing.
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data ({@link ParquetChunkData})
  */
 public interface Flusher<T> {
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 9c35a13df..03b1c1762 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -23,7 +23,6 @@
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
-import org.apache.arrow.memory.BufferAllocator;
 import org.apache.parquet.hadoop.BdecParquetWriter;
 import org.apache.parquet.schema.MessageType;
 import org.apache.parquet.schema.PrimitiveType;
@@ -59,7 +58,6 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
   ParquetRowBuffer(
       OpenChannelRequest.OnErrorOption onErrorOption,
       ZoneId defaultTimezone,
-      BufferAllocator allocator,
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelRuntimeState,
@@ -68,7 +66,6 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
     super(
         onErrorOption,
         defaultTimezone,
-        allocator,
         fullyQualifiedChannelName,
         rowSizeMetric,
         channelRuntimeState);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index 44b90987b..1e3e63d98 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -25,8 +25,7 @@
  * Register one or more blobs to the targeted Snowflake table, it will be done using the dedicated
  * thread in order to maintain ordering per channel
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data ({@link ParquetChunkData})
  */
 class RegisterService<T> {
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
index 5b7e937cd..d7ba6dbd9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
@@ -12,8 +12,7 @@
  * Interface for the buffer in the Streaming Ingest channel that holds the un-flushed rows, these
  * rows will be converted to the underlying format implementation for faster processing
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data ({@link ParquetChunkData})
  */
 interface RowBuffer<T> {
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java
index ed75796a4..3e442d43b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java
@@ -7,8 +7,6 @@
 import java.time.ZoneId;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
 
 /** Builds a Streaming Ingest channel for a specific Streaming Ingest client */
 class SnowflakeStreamingIngestChannelFactory {
@@ -105,7 +103,6 @@ SnowflakeStreamingIngestChannelInternal<T> build() {
       Utils.assertNotNull("encryption key_id", this.encryptionKeyId);
       Utils.assertNotNull("on_error option", this.onErrorOption);
       Utils.assertNotNull("default timezone", this.defaultTimezone);
-      BufferAllocator allocator = createBufferAllocator();
       return new SnowflakeStreamingIngestChannelInternal<>(
           this.name,
           this.dbName,
@@ -119,19 +116,7 @@ SnowflakeStreamingIngestChannelInternal<T> build() {
           this.encryptionKeyId,
           this.onErrorOption,
           this.defaultTimezone,
-          this.owningClient.getParameterProvider().getBlobFormatVersion(),
-          allocator);
-    }
-
-    private BufferAllocator createBufferAllocator() {
-      return owningClient.isTestMode()
-          ? new RootAllocator()
-          : owningClient
-              .getAllocator()
-              .newChildAllocator(
-                  String.format("%s_%s", name, channelSequencer),
-                  0,
-                  owningClient.getAllocator().getLimit());
+          this.owningClient.getParameterProvider().getBlobFormatVersion());
     }
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index f4c4c6952..136713324 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -27,14 +27,11 @@
 import net.snowflake.ingest.utils.ParameterProvider;
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
 
 /**
  * The first version of implementation for SnowflakeStreamingIngestChannel
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data {@link ParquetChunkData})
  */
 class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIngestChannel {
 
@@ -93,8 +90,7 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
         encryptionKeyId,
         onErrorOption,
         defaultTimezone,
-        client.getParameterProvider().getBlobFormatVersion(),
-        new RootAllocator());
+        client.getParameterProvider().getBlobFormatVersion());
   }
 
   /** Default constructor */
@@ -111,8 +107,7 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
       Long encryptionKeyId,
       OpenChannelRequest.OnErrorOption onErrorOption,
       ZoneId defaultTimezone,
-      Constants.BdecVersion bdecVersion,
-      BufferAllocator allocator) {
+      Constants.BdecVersion bdecVersion) {
     this.isClosed = false;
     this.owningClient = client;
     this.channelFlushContext =
@@ -123,7 +118,6 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
         AbstractRowBuffer.createRowBuffer(
             onErrorOption,
             defaultTimezone,
-            allocator,
             bdecVersion,
             getFullyQualifiedName(),
             this::collectRowSize,
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 4a87e2bd2..ec396c6d3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -36,6 +36,7 @@
 import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
 import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
 import java.security.KeyPair;
 import java.security.NoSuchAlgorithmException;
@@ -73,9 +74,6 @@
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.SnowflakeURL;
 import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.util.VisibleForTesting;
 
 /**
  * The first version of implementation for SnowflakeStreamingIngestClient. The client internally
@@ -83,8 +81,7 @@
  * <li>the channel cache, which contains all the channels that belong to this account
  * <li>the flush service, which schedules and coordinates the flush to Snowflake tables
  *
- * @param <T> type of column data (Arrow {@link org.apache.arrow.vector.VectorSchemaRoot} or {@link
- *     ParquetChunkData})
+ * @param <T> type of column data ({@link ParquetChunkData})
  */
 public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStreamingIngestClient {
 
@@ -116,9 +113,6 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
   // Reference to the flush service
   private final FlushService<T> flushService;
 
-  // Memory allocator
-  private final BufferAllocator allocator;
-
   // Indicates whether the client has closed
   private volatile boolean isClosed;
 
@@ -172,7 +166,6 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
     this.isTestMode = isTestMode;
     this.httpClient = httpClient == null ? HttpUtil.getHttpClient(accountName) : httpClient;
     this.channelCache = new ChannelCache<>();
-    this.allocator = new RootAllocator();
     this.isClosed = false;
     this.requestBuilder = requestBuilder;
 
@@ -632,7 +625,6 @@ public void close() throws Exception {
         this.requestBuilder.closeResources();
       }
       HttpUtil.shutdownHttpConnectionManagerDaemonThread();
-      Utils.closeAllocator(this.allocator);
     }
   }
 
@@ -654,15 +646,6 @@ void setNeedFlush() {
     this.flushService.setNeedFlush();
   }
 
-  /**
-   * Get the buffer allocator
-   *
-   * @return the buffer allocator
-   */
-  BufferAllocator getAllocator() {
-    return this.allocator;
-  }
-
   /** Remove the channel in the channel cache if the channel sequencer matches */
   void removeChannelIfSequencersMatch(SnowflakeStreamingIngestChannelInternal<T> channel) {
     this.channelCache.removeChannelIfSequencersMatch(channel);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index 2feed6b81..0d7e3f211 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -11,6 +11,7 @@
 import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
 import static net.snowflake.ingest.utils.HttpUtil.generateProxyPropertiesForJDBC;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
@@ -39,7 +40,6 @@
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.util.VisibleForTesting;
 
 /** Handles uploading files to the Snowflake Streaming Ingest Stage */
 class StreamingIngestStage {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/TimestampWrapper.java b/src/main/java/net/snowflake/ingest/streaming/internal/TimestampWrapper.java
index deb664edd..871cf8c14 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/TimestampWrapper.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/TimestampWrapper.java
@@ -12,7 +12,7 @@
 
 /**
  * This class represents the outcome of timestamp parsing and validation. It contains methods needed
- * to serialize timestamps into Arrow and Parquet.
+ * to serialize timestamps into Parquet.
  */
 public class TimestampWrapper {
 
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index 050f14264..41465304a 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -61,10 +61,10 @@ public enum WriteMode {
     REST_API,
   }
 
-  /** The write mode to generate Arrow BDEC file. */
+  /** The write mode to generate BDEC file. */
   public enum BdecVersion {
-    /** Uses Arrow to generate BDEC chunks. */
-    ONE(1),
+    // Unused (previously Arrow)
+    // ONE(1),
 
     // Unused (previously Arrow with per column compression.
     // TWO(2),
diff --git a/src/main/java/net/snowflake/ingest/utils/Cryptor.java b/src/main/java/net/snowflake/ingest/utils/Cryptor.java
index 74a033fe1..ae7199417 100644
--- a/src/main/java/net/snowflake/ingest/utils/Cryptor.java
+++ b/src/main/java/net/snowflake/ingest/utils/Cryptor.java
@@ -2,6 +2,7 @@
 
 import static net.snowflake.ingest.utils.Constants.ENCRYPTION_ALGORITHM;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.security.InvalidAlgorithmParameterException;
@@ -16,7 +17,6 @@
 import javax.crypto.SecretKey;
 import javax.crypto.spec.IvParameterSpec;
 import javax.crypto.spec.SecretKeySpec;
-import org.apache.arrow.util.VisibleForTesting;
 
 public class Cryptor {
 
diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
index e2f38f476..7faea2859 100644
--- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
+++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -209,7 +209,7 @@ public boolean hasEnabledSnowpipeStreamingMetrics() {
     return (boolean) val;
   }
 
-  /** @return Blob format version: 1 (arrow stream write mode), 2 (arrow file write mode) etc */
+  /** @return Blob format version */
   public Constants.BdecVersion getBlobFormatVersion() {
     Object val = this.parameterMap.getOrDefault(BLOB_FORMAT_VERSION, BLOB_FORMAT_VERSION_DEFAULT);
     if (val instanceof Constants.BdecVersion) {
diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java
index 1280dbf46..662063a90 100644
--- a/src/main/java/net/snowflake/ingest/utils/Utils.java
+++ b/src/main/java/net/snowflake/ingest/utils/Utils.java
@@ -33,7 +33,6 @@
 import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.operator.InputDecryptorProvider;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo;
-import org.apache.arrow.memory.BufferAllocator;
 import org.apache.commons.codec.binary.Base64;
 
 /** Contains Ingest related utility functions */
@@ -268,14 +267,6 @@ public static boolean isNullOrEmpty(String string) {
     return string == null || string.isEmpty();
   }
 
-  /** Release any outstanding memory and then close the buffer allocator */
-  public static void closeAllocator(BufferAllocator alloc) {
-    for (BufferAllocator childAlloc : alloc.getChildAllocators()) {
-      childAlloc.close();
-    }
-    alloc.close();
-  }
-
   /** Util function to show memory usage info and debug memory issue in the SDK */
   public static void showMemory() {
     List<BufferPoolMXBean> pools = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class);
diff --git a/src/main/resources/META-INF/third-party-licenses/REVISED_BSD.txt b/src/main/resources/META-INF/third-party-licenses/REVISED_BSD.txt
deleted file mode 100644
index 641d4f45e..000000000
--- a/src/main/resources/META-INF/third-party-licenses/REVISED_BSD.txt
+++ /dev/null
@@ -1,30 +0,0 @@
-JSch 0.0.* was released under the GNU LGPL license.  Later, we have switched
-over to a BSD-style license.
-
-------------------------------------------------------------------------------
-Copyright (c) 2002-2015 Atsuhiko Yamanaka, JCraft,Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-  1. Redistributions of source code must retain the above copyright notice,
-     this list of conditions and the following disclaimer.
-
-  2. Redistributions in binary form must reproduce the above copyright
-     notice, this list of conditions and the following disclaimer in
-     the documentation and/or other materials provided with the distribution.
-
-  3. The names of the authors may not be used to endorse or promote products
-     derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
-INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
-FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT,
-INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
-INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
-OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
-EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 293ece5bf..e0d7a3e07 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -30,8 +30,6 @@
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
-import java.util.Arrays;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
@@ -148,14 +146,6 @@ private static String getTestProfilePath() {
     return testProfilePath;
   }
 
-  /** @return list of Bdec versions for which to execute IT tests. */
-  public static Collection<Object[]> getBdecVersionItCases() {
-    return Arrays.asList(
-        new Object[][] {
-          {"Arrow", Constants.BdecVersion.ONE}, {"Parquet", Constants.BdecVersion.THREE}
-        });
-  }
-
   public static String getUser() throws Exception {
     if (profile == null) {
       init();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
deleted file mode 100644
index f2cc72a16..000000000
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ArrowBufferTest.java
+++ /dev/null
@@ -1,444 +0,0 @@
-package net.snowflake.ingest.streaming.internal;
-
-import static net.snowflake.ingest.streaming.internal.ArrowRowBuffer.DECIMAL_BIT_WIDTH;
-
-import java.time.ZoneOffset;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import net.snowflake.ingest.streaming.InsertValidationResponse;
-import net.snowflake.ingest.streaming.OpenChannelRequest;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.types.Types;
-import org.apache.arrow.vector.types.pojo.ArrowType;
-import org.apache.arrow.vector.types.pojo.Field;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class ArrowBufferTest {
-  private ArrowRowBuffer rowBufferOnErrorContinue;
-
-  @Before
-  public void setupRowBuffer() {
-    this.rowBufferOnErrorContinue = createTestBuffer(OpenChannelRequest.OnErrorOption.CONTINUE);
-    this.rowBufferOnErrorContinue.setupSchema(RowBufferTest.createSchema());
-  }
-
-  ArrowRowBuffer createTestBuffer(OpenChannelRequest.OnErrorOption onErrorOption) {
-    ChannelRuntimeState initialState = new ChannelRuntimeState("0", 0L, true);
-    return new ArrowRowBuffer(
-        onErrorOption, ZoneOffset.UTC, new RootAllocator(), "test.buffer", rs -> {}, initialState);
-  }
-
-  @Test
-  public void testFieldNumberAfterFlush() {
-    String offsetToken = "1";
-    Map<String, Object> row = new HashMap<>();
-    row.put("colTinyInt", (byte) 1);
-    row.put("\"colTinyInt\"", (byte) 1);
-    row.put("colSmallInt", (short) 2);
-    row.put("colInt", 3);
-    row.put("colBigInt", 4L);
-    row.put("colDecimal", 1.23);
-    row.put("colChar", "2");
-
-    InsertValidationResponse response =
-        rowBufferOnErrorContinue.insertRows(Collections.singletonList(row), offsetToken);
-    Assert.assertFalse(response.hasErrors());
-
-    ChannelData<VectorSchemaRoot> data =
-        rowBufferOnErrorContinue.flush("my_snowpipe_streaming.bdec");
-    Assert.assertEquals(7, data.getVectors().getFieldVectors().size());
-  }
-
-  @Test
-  public void buildFieldFixedSB1() {
-    // FIXED, SB1
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB1")
-            .nullable(true)
-            .build();
-
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("TESTCOL", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.TINYINT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB1");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "FIXED");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("nullable"), "true");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldFixedSB2() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB2")
-            .nullable(false)
-            .build();
-
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("TESTCOL", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.SMALLINT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB2");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "FIXED");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("nullable"), "false");
-    Assert.assertFalse(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldFixedSB4() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB4")
-            .nullable(true)
-            .build();
-
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("TESTCOL", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.INT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB4");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "FIXED");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldFixedSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("TESTCOL", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.BIGINT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "FIXED");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldFixedSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("FIXED")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    ArrowType expectedType =
-        new ArrowType.Decimal(testCol.getPrecision(), testCol.getScale(), DECIMAL_BIT_WIDTH);
-
-    Assert.assertEquals("TESTCOL", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), expectedType);
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB16");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "FIXED");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldLobVariant() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("VARIANT")
-            .physicalType("LOB")
-            .nullable(true)
-            .build();
-
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("TESTCOL", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.VARCHAR.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "LOB");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "VARIANT");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldTimestampNtzSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_NTZ")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("TESTCOL", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.BIGINT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIMESTAMP_NTZ");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldTimestampNtzSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_NTZ")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("TESTCOL", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.STRUCT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB16");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIMESTAMP_NTZ");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 2);
-    Assert.assertEquals(
-        result.getChildren().get(0).getFieldType().getType(), Types.MinorType.BIGINT.getType());
-    Assert.assertEquals(
-        result.getChildren().get(1).getFieldType().getType(), Types.MinorType.INT.getType());
-  }
-
-  @Test
-  public void buildFieldTimestampTzSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_TZ")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("TESTCOL", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.STRUCT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIMESTAMP_TZ");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 2);
-    Assert.assertEquals(
-        result.getChildren().get(0).getFieldType().getType(), Types.MinorType.BIGINT.getType());
-    Assert.assertEquals(
-        result.getChildren().get(1).getFieldType().getType(), Types.MinorType.INT.getType());
-  }
-
-  @Test
-  public void buildFieldTimestampTzSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIMESTAMP_TZ")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("TESTCOL", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.STRUCT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB16");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIMESTAMP_TZ");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 3);
-    Assert.assertEquals(
-        result.getChildren().get(0).getFieldType().getType(), Types.MinorType.BIGINT.getType());
-    Assert.assertEquals(
-        result.getChildren().get(1).getFieldType().getType(), Types.MinorType.INT.getType());
-    Assert.assertEquals(
-        result.getChildren().get(2).getFieldType().getType(), Types.MinorType.INT.getType());
-  }
-
-  @Test
-  public void buildFieldTimestampDate() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("DATE")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("TESTCOL", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.DATEDAY.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "DATE");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldTimeSB4() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIME")
-            .physicalType("SB4")
-            .nullable(true)
-            .build();
-
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("TESTCOL", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.INT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB4");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIME");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldTimeSB8() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("TIME")
-            .physicalType("SB8")
-            .nullable(true)
-            .build();
-
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("TESTCOL", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.BIGINT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB8");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "TIME");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldBoolean() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("BOOLEAN")
-            .physicalType("BINARY")
-            .nullable(true)
-            .build();
-
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("TESTCOL", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.BIT.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "BINARY");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "BOOLEAN");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void buildFieldRealSB16() {
-    ColumnMetadata testCol =
-        ColumnMetadataBuilder.newBuilder()
-            .logicalType("REAL")
-            .physicalType("SB16")
-            .nullable(true)
-            .build();
-
-    Field result = this.rowBufferOnErrorContinue.buildField(testCol);
-
-    Assert.assertEquals("TESTCOL", result.getName());
-    Assert.assertEquals(result.getFieldType().getType(), Types.MinorType.FLOAT8.getType());
-    Assert.assertEquals(result.getFieldType().getMetadata().get("physicalType"), "SB16");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("scale"), "0");
-    Assert.assertEquals(result.getFieldType().getMetadata().get("logicalType"), "REAL");
-    Assert.assertTrue(result.getFieldType().isNullable());
-    Assert.assertEquals(result.getChildren().size(), 0);
-  }
-
-  @Test
-  public void testArrowE2ETimestampLTZ() {
-    testArrowE2ETimestampLTZHelper(OpenChannelRequest.OnErrorOption.ABORT);
-    testArrowE2ETimestampLTZHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
-  }
-
-  private void testArrowE2ETimestampLTZHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
-    ArrowRowBuffer innerBuffer = createTestBuffer(onErrorOption);
-
-    ColumnMetadata colTimestampLtzSB8 = new ColumnMetadata();
-    colTimestampLtzSB8.setName("COLTIMESTAMPLTZ_SB8");
-    colTimestampLtzSB8.setPhysicalType("SB8");
-    colTimestampLtzSB8.setNullable(false);
-    colTimestampLtzSB8.setLogicalType("TIMESTAMP_LTZ");
-    colTimestampLtzSB8.setScale(0);
-
-    ColumnMetadata colTimestampLtzSB16 = new ColumnMetadata();
-    colTimestampLtzSB16.setName("COLTIMESTAMPLTZ_SB16");
-    colTimestampLtzSB16.setPhysicalType("SB16");
-    colTimestampLtzSB16.setNullable(false);
-    colTimestampLtzSB16.setLogicalType("TIMESTAMP_LTZ");
-    colTimestampLtzSB16.setScale(9);
-
-    innerBuffer.setupSchema(Arrays.asList(colTimestampLtzSB8, colTimestampLtzSB16));
-
-    Map<String, Object> row = new HashMap<>();
-    row.put("COLTIMESTAMPLTZ_SB8", "1621899220");
-    row.put("COLTIMESTAMPLTZ_SB16", "1621899220123456789");
-
-    InsertValidationResponse response =
-        innerBuffer.insertRows(Collections.singletonList(row), null);
-    Assert.assertFalse(response.hasErrors());
-    Assert.assertEquals(
-        1621899220L, innerBuffer.vectorsRoot.getVector("COLTIMESTAMPLTZ_SB8").getObject(0));
-    Assert.assertEquals(
-        "epoch",
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPLTZ_SB16")
-            .getChildrenFromFields()
-            .get(0)
-            .getName());
-    Assert.assertEquals(
-        1621899220L,
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPLTZ_SB16")
-            .getChildrenFromFields()
-            .get(0)
-            .getObject(0));
-    Assert.assertEquals(
-        "fraction",
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPLTZ_SB16")
-            .getChildrenFromFields()
-            .get(1)
-            .getName());
-    Assert.assertEquals(
-        123456789,
-        innerBuffer
-            .vectorsRoot
-            .getVector("COLTIMESTAMPLTZ_SB16")
-            .getChildrenFromFields()
-            .get(1)
-            .getObject(0));
-  }
-}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 12e77ab7b..fc049c297 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -28,7 +28,6 @@
 import java.util.Arrays;
 import java.util.Base64;
 import java.util.Calendar;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -46,28 +45,16 @@
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.ParameterProvider;
 import net.snowflake.ingest.utils.SFException;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 import org.mockito.ArgumentCaptor;
 import org.mockito.ArgumentMatchers;
 import org.mockito.Mockito;
 
-@RunWith(Parameterized.class)
 public class FlushServiceTest {
-  @Parameterized.Parameters(name = "{0}")
-  public static Collection<Object[]> testContextFactory() {
-    return Arrays.asList(
-        new Object[][] {{ArrowTestContext.createFactory()}, {ParquetTestContext.createFactory()}});
-  }
-
-  public FlushServiceTest(TestContextFactory<?> testContextFactory) {
-    this.testContextFactory = testContextFactory;
+  public FlushServiceTest() {
+    this.testContextFactory = ParquetTestContext.createFactory();
   }
 
   private abstract static class TestContextFactory<T> {
@@ -244,58 +231,6 @@ static RowSetBuilder newBuilder() {
     }
   }
 
-  private static class ArrowTestContext extends TestContext<VectorSchemaRoot> {
-    private final BufferAllocator allocator = new RootAllocator();
-
-    SnowflakeStreamingIngestChannelInternal<VectorSchemaRoot> createChannel(
-        String name,
-        String dbName,
-        String schemaName,
-        String tableName,
-        String offsetToken,
-        Long channelSequencer,
-        Long rowSequencer,
-        String encryptionKey,
-        Long encryptionKeyId,
-        OpenChannelRequest.OnErrorOption onErrorOption,
-        ZoneId defaultTimezone) {
-      return new SnowflakeStreamingIngestChannelInternal<>(
-          name,
-          dbName,
-          schemaName,
-          tableName,
-          offsetToken,
-          channelSequencer,
-          rowSequencer,
-          client,
-          encryptionKey,
-          encryptionKeyId,
-          onErrorOption,
-          defaultTimezone,
-          Constants.BdecVersion.ONE,
-          allocator);
-    }
-
-    @Override
-    public void close() {
-      try {
-        // Close allocator to make sure no memory leak
-        allocator.close();
-      } catch (Exception e) {
-        Assert.fail(String.format("Allocator close failure. Caused by %s", e.getMessage()));
-      }
-    }
-
-    static TestContextFactory<VectorSchemaRoot> createFactory() {
-      return new TestContextFactory<VectorSchemaRoot>("Arrow") {
-        @Override
-        TestContext<VectorSchemaRoot> create() {
-          return new ArrowTestContext();
-        }
-      };
-    }
-  }
-
   private static class ParquetTestContext extends TestContext<List<List<Object>>> {
 
     SnowflakeStreamingIngestChannelInternal<List<List<Object>>> createChannel(
@@ -323,8 +258,7 @@ SnowflakeStreamingIngestChannelInternal<List<List<Object>>> createChannel(
           encryptionKeyId,
           onErrorOption,
           defaultTimezone,
-          Constants.BdecVersion.THREE,
-          null);
+          Constants.BdecVersion.THREE);
     }
 
     @Override
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/OpenManyChannelsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/OpenManyChannelsIT.java
index c3752a660..1934b3f77 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/OpenManyChannelsIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/OpenManyChannelsIT.java
@@ -51,7 +51,7 @@ public void setUp() throws Exception {
             String.format(
                 "create or replace table %s.%s.%s (col int)",
                 databaseName, SCHEMA_NAME, TABLE_NAME));
-    Properties props = TestUtils.getProperties(Constants.BdecVersion.ONE);
+    Properties props = TestUtils.getProperties(Constants.BdecVersion.THREE);
     if (props.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       props.setProperty(ROLE, "ACCOUNTADMIN");
     }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index b6143e6ce..b6e035c90 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -8,7 +8,6 @@
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -20,34 +19,18 @@
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
-import org.apache.arrow.memory.RootAllocator;
-import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.lang3.NotImplementedException;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 
-@RunWith(Parameterized.class)
 public class RowBufferTest {
-  @Parameterized.Parameters(name = "{0}")
-  public static Collection<Object[]> bdecVersion() {
-    return Arrays.asList(
-        new Object[][] {
-          {"Arrow", Constants.BdecVersion.ONE},
-          {"Parquet_w/o_optimization", Constants.BdecVersion.THREE}
-        });
-  }
 
-  private final Constants.BdecVersion bdecVersion;
   private final boolean enableParquetMemoryOptimization;
   private AbstractRowBuffer<?> rowBufferOnErrorContinue;
   private AbstractRowBuffer<?> rowBufferOnErrorAbort;
 
-  public RowBufferTest(@SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
-    this.bdecVersion = bdecVersion;
+  public RowBufferTest() {
     this.enableParquetMemoryOptimization = false;
   }
 
@@ -127,8 +110,7 @@ private AbstractRowBuffer<?> createTestBuffer(OpenChannelRequest.OnErrorOption o
     return AbstractRowBuffer.createRowBuffer(
         onErrorOption,
         UTC,
-        new RootAllocator(),
-        bdecVersion,
+        Constants.BdecVersion.THREE,
         "test.buffer",
         rs -> {},
         initialState,
@@ -493,12 +475,10 @@ private void testFlushHelper(AbstractRowBuffer<?> rowBuffer) {
     Assert.assertEquals(offsetToken, data.getOffsetToken());
     Assert.assertEquals(bufferSize, data.getBufferSize(), 0);
 
-    if (bdecVersion == Constants.BdecVersion.THREE) {
-      final ParquetChunkData chunkData = (ParquetChunkData) data.getVectors();
-      Assert.assertEquals(
-          StreamingIngestUtils.getShortname(filename),
-          chunkData.metadata.get(Constants.PRIMARY_FILE_ID_KEY));
-    }
+    final ParquetChunkData chunkData = (ParquetChunkData) data.getVectors();
+    Assert.assertEquals(
+        StreamingIngestUtils.getShortname(filename),
+        chunkData.metadata.get(Constants.PRIMARY_FILE_ID_KEY));
   }
 
   @Test
@@ -765,10 +745,8 @@ private void testStatsE2EHelper(AbstractRowBuffer<?> rowBuffer) {
     Assert.assertEquals(0, columnEpStats.get("COLCHAR").getCurrentNullCount());
     Assert.assertEquals(-1, columnEpStats.get("COLCHAR").getDistinctValues());
 
-    if (bdecVersion == Constants.BdecVersion.THREE) {
-      final ParquetChunkData chunkData = (ParquetChunkData) result.getVectors();
-      Assert.assertEquals(filename, chunkData.metadata.get(Constants.PRIMARY_FILE_ID_KEY));
-    }
+    final ParquetChunkData chunkData = (ParquetChunkData) result.getVectors();
+    Assert.assertEquals(filename, chunkData.metadata.get(Constants.PRIMARY_FILE_ID_KEY));
 
     // Confirm we reset
     ChannelData<?> resetResults = rowBuffer.flush("my_snowpipe_streaming.bdec");
@@ -1563,42 +1541,17 @@ public void testOnErrorAbortRowsWithError() {
     Assert.assertThrows(
         SFException.class, () -> innerBufferOnErrorAbort.insertRows(mixedRows, "3"));
 
-    switch (bdecVersion) {
-      case ONE:
-        VectorSchemaRoot snapshotContinueArrow =
-            ((VectorSchemaRoot) innerBufferOnErrorContinue.getSnapshot("fake/filePath").get());
-        // validRows and only the good row from mixedRows are in the buffer
-        Assert.assertEquals(2, snapshotContinueArrow.getRowCount());
-        Assert.assertEquals("[a, b]", snapshotContinueArrow.getVector(0).toString());
-
-        VectorSchemaRoot snapshotAbortArrow =
-            ((VectorSchemaRoot) innerBufferOnErrorAbort.getSnapshot("fake/filePath").get());
-        // only validRows and none of the mixedRows are in the buffer
-        Assert.assertEquals(1, snapshotAbortArrow.getRowCount());
-        Assert.assertEquals("[a]", snapshotAbortArrow.getVector(0).toString());
-        break;
-
-      case THREE:
-        List<List<Object>> snapshotContinueParquet =
-            ((ParquetChunkData) innerBufferOnErrorContinue.getSnapshot("fake/filePath").get()).rows;
-        // validRows and only the good row from mixedRows are in the buffer
-        Assert.assertEquals(2, snapshotContinueParquet.size());
-        Assert.assertEquals(Arrays.asList("a"), snapshotContinueParquet.get(0));
-        Assert.assertEquals(Arrays.asList("b"), snapshotContinueParquet.get(1));
-
-        List<List<Object>> snapshotAbortParquet =
-            ((ParquetChunkData) innerBufferOnErrorAbort.getSnapshot("fake/filePath").get()).rows;
-        // only validRows and none of the mixedRows are in the buffer
-        Assert.assertEquals(1, snapshotAbortParquet.size());
-        Assert.assertEquals(Arrays.asList("a"), snapshotAbortParquet.get(0));
-        break;
-      default:
-        throw new NotImplementedException("Unsupported version!");
-    }
-    if (bdecVersion == Constants.BdecVersion.THREE) {
-
-    } else if (bdecVersion == Constants.BdecVersion.ONE) {
-
-    }
+    List<List<Object>> snapshotContinueParquet =
+        ((ParquetChunkData) innerBufferOnErrorContinue.getSnapshot("fake/filePath").get()).rows;
+    // validRows and only the good row from mixedRows are in the buffer
+    Assert.assertEquals(2, snapshotContinueParquet.size());
+    Assert.assertEquals(Arrays.asList("a"), snapshotContinueParquet.get(0));
+    Assert.assertEquals(Arrays.asList("b"), snapshotContinueParquet.get(1));
+
+    List<List<Object>> snapshotAbortParquet =
+        ((ParquetChunkData) innerBufferOnErrorAbort.getSnapshot("fake/filePath").get()).rows;
+    // only validRows and none of the mixedRows are in the buffer
+    Assert.assertEquals(1, snapshotAbortParquet.size());
+    Assert.assertEquals(Arrays.asList("a"), snapshotAbortParquet.get(0));
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index d34386bfa..23c95d792 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -38,7 +38,6 @@
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.SnowflakeURL;
 import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.vector.VectorSchemaRoot;
 import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -498,14 +497,7 @@ public void testOpenChannelSuccessResponse() throws Exception {
   @Test
   public void testInsertRow() {
     SnowflakeStreamingIngestClientInternal<?> client;
-    boolean isArrowDefault =
-        ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT == Constants.BdecVersion.ONE;
-    if (isArrowDefault) {
-      client = new SnowflakeStreamingIngestClientInternal<VectorSchemaRoot>("client_ARROW");
-    } else {
-      client = new SnowflakeStreamingIngestClientInternal<ParquetChunkData>("client_PARQUET");
-    }
-
+    client = new SnowflakeStreamingIngestClientInternal<ParquetChunkData>("client_PARQUET");
     SnowflakeStreamingIngestChannelInternal<?> channel =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel",
@@ -550,11 +542,7 @@ public void testInsertRow() {
     data = channel.getData("my_snowpipe_streaming.bdec");
     Assert.assertEquals(2, data.getRowCount());
     Assert.assertEquals((Long) 1L, data.getRowSequencer());
-    Assert.assertEquals(
-        1,
-        isArrowDefault
-            ? ((ChannelData<VectorSchemaRoot>) data).getVectors().getFieldVectors().size()
-            : ((ChannelData<ParquetChunkData>) data).getVectors().rows.get(0).size());
+    Assert.assertEquals(1, ((ChannelData<ParquetChunkData>) data).getVectors().rows.get(0).size());
     Assert.assertEquals("2", data.getOffsetToken());
     Assert.assertTrue(data.getBufferSize() > 0);
     Assert.assertTrue(insertStartTimeInMs <= data.getMinMaxInsertTimeInMs().getFirst());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index 3b50c8354..244457ccf 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -59,7 +59,6 @@
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.SnowflakeURL;
 import net.snowflake.ingest.utils.Utils;
-import org.apache.arrow.memory.RootAllocator;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -69,6 +68,8 @@
 public class SnowflakeStreamingIngestClientTest {
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
+  private static final Constants.BdecVersion BDEC_VERSION = Constants.BdecVersion.THREE;
+
   SnowflakeStreamingIngestChannelInternal<StubChunkData> channel1;
   SnowflakeStreamingIngestChannelInternal<StubChunkData> channel2;
   SnowflakeStreamingIngestChannelInternal<StubChunkData> channel3;
@@ -92,8 +93,7 @@ public void setup() {
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
             ZoneOffset.UTC,
-            Constants.BdecVersion.ONE,
-            new RootAllocator());
+            BDEC_VERSION);
     channel2 =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel2",
@@ -108,8 +108,7 @@ public void setup() {
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
             ZoneOffset.UTC,
-            Constants.BdecVersion.ONE,
-            new RootAllocator());
+            BDEC_VERSION);
     channel3 =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel3",
@@ -124,8 +123,7 @@ public void setup() {
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
             ZoneOffset.UTC,
-            Constants.BdecVersion.ONE,
-            new RootAllocator());
+            BDEC_VERSION);
     channel4 =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel4",
@@ -140,8 +138,7 @@ public void setup() {
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
             ZoneOffset.UTC,
-            Constants.BdecVersion.ONE,
-            new RootAllocator());
+            BDEC_VERSION);
   }
 
   @Test
@@ -358,8 +355,7 @@ public void testGetChannelsStatusWithRequest() throws Exception {
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
             ZoneOffset.UTC,
-            Constants.BdecVersion.ONE,
-            new RootAllocator());
+            BDEC_VERSION);
 
     ChannelsStatusRequest.ChannelStatusRequestDTO dto =
         new ChannelsStatusRequest.ChannelStatusRequestDTO(channel);
@@ -418,8 +414,7 @@ public void testGetChannelsStatusWithRequestError() throws Exception {
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
             ZoneOffset.UTC,
-            Constants.BdecVersion.ONE,
-            new RootAllocator());
+            BDEC_VERSION);
 
     try {
       client.getChannelsStatus(Collections.singletonList(channel));
@@ -460,8 +455,7 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
             ZoneOffset.UTC,
-            Constants.BdecVersion.ONE,
-            new RootAllocator());
+            BDEC_VERSION);
 
     ChannelMetadata channelMetadata =
         ChannelMetadata.builder()
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
index f12484062..4d7e71ac7 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
@@ -17,17 +17,9 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 
 /** Ingest large amount of rows. */
-@RunWith(Parameterized.class)
 public class StreamingIngestBigFilesIT {
-  @Parameterized.Parameters(name = "{0}")
-  public static Collection<Object[]> bdecVersion() {
-    return TestUtils.getBdecVersionItCases();
-  }
-
   private static final String TEST_DB_PREFIX = "STREAMING_INGEST_TEST_DB";
   private static final String TEST_SCHEMA = "STREAMING_INGEST_TEST_SCHEMA";
 
@@ -37,13 +29,6 @@ public static Collection<Object[]> bdecVersion() {
   private Connection jdbcConnection;
   private String testDb;
 
-  private final Constants.BdecVersion bdecVersion;
-
-  public StreamingIngestBigFilesIT(
-      @SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
-    this.bdecVersion = bdecVersion;
-  }
-
   @Before
   public void beforeAll() throws Exception {
     testDb = TEST_DB_PREFIX + "_" + UUID.randomUUID().toString().substring(0, 4);
@@ -62,7 +47,7 @@ public void beforeAll() throws Exception {
         .createStatement()
         .execute(String.format("use warehouse %s", TestUtils.getWarehouse()));
 
-    prop = TestUtils.getProperties(bdecVersion);
+    prop = TestUtils.getProperties(Constants.BdecVersion.THREE);
     if (prop.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       prop.setProperty(ROLE, "ACCOUNTADMIN");
     }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 1f9ea3914..2bcfbcdb2 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -16,7 +16,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -47,19 +46,11 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 import org.mockito.ArgumentMatchers;
 import org.mockito.Mockito;
 
 /** Example streaming ingest sdk integration test */
-@RunWith(Parameterized.class)
 public class StreamingIngestIT {
-  @Parameterized.Parameters(name = "{0}")
-  public static Collection<Object[]> bdecVersion() {
-    return TestUtils.getBdecVersionItCases();
-  }
-
   private static final String TEST_TABLE = "STREAMING_INGEST_TEST_TABLE";
   private static final String TEST_DB_PREFIX = "STREAMING_INGEST_TEST_DB";
   private static final String TEST_SCHEMA = "STREAMING_INGEST_TEST_SCHEMA";
@@ -74,13 +65,6 @@ public static Collection<Object[]> bdecVersion() {
   private Connection jdbcConnection;
   private String testDb;
 
-  private final Constants.BdecVersion bdecVersion;
-
-  public StreamingIngestIT(
-      @SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
-    this.bdecVersion = bdecVersion;
-  }
-
   @Before
   public void beforeAll() throws Exception {
     testDb = TEST_DB_PREFIX + "_" + UUID.randomUUID().toString().substring(0, 4);
@@ -105,7 +89,7 @@ public void beforeAll() throws Exception {
         .createStatement()
         .execute(String.format("use warehouse %s", TestUtils.getWarehouse()));
 
-    prop = TestUtils.getProperties(bdecVersion);
+    prop = TestUtils.getProperties(Constants.BdecVersion.THREE);
     if (prop.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       prop.setProperty(ROLE, "ACCOUNTADMIN");
     }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index 5ef3951c1..4ff61e142 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -10,7 +10,6 @@
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.time.ZoneId;
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
@@ -27,16 +26,8 @@
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 
-@RunWith(Parameterized.class)
 public abstract class AbstractDataTypeTest {
-  @Parameterized.Parameters(name = "{0}")
-  public static Collection<Object[]> bdecVersion() {
-    return TestUtils.getBdecVersionItCases();
-  }
-
   private static final String SOURCE_COLUMN_NAME = "source";
   private static final String VALUE_COLUMN_NAME = "value";
 
@@ -65,13 +56,6 @@ public static Collection<Object[]> bdecVersion() {
   private SnowflakeStreamingIngestClient client;
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
-  private final Constants.BdecVersion bdecVersion;
-
-  public AbstractDataTypeTest(
-      @SuppressWarnings("unused") String name, Constants.BdecVersion bdecVersion) {
-    this.bdecVersion = bdecVersion;
-  }
-
   @Before
   public void before() throws Exception {
     databaseName = String.format("SDK_DATATYPE_COMPATIBILITY_IT_%s", getRandomIdentifier());
@@ -82,7 +66,7 @@ public void before() throws Exception {
 
     conn.createStatement().execute(String.format("use warehouse %s;", TestUtils.getWarehouse()));
 
-    Properties props = TestUtils.getProperties(bdecVersion);
+    Properties props = TestUtils.getProperties(Constants.BdecVersion.THREE);
     if (props.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       props.setProperty(ROLE, "ACCOUNTADMIN");
     }
@@ -180,7 +164,7 @@ protected <T> void expectNumberOutOfRangeError(
                             maxPowerOf10Exclusive, maxPowerOf10Exclusive))));
   }
 
-  protected <T> void expectArrowNotSupported(String dataType, T value) throws Exception {
+  protected <T> void expectNotSupported(String dataType, T value) throws Exception {
     expectError(
         dataType,
         value,
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
index d79d6836b..d9e6e3ddd 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
@@ -1,15 +1,10 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
 import net.snowflake.client.jdbc.internal.org.bouncycastle.util.encoders.Hex;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class BinaryIT extends AbstractDataTypeTest {
 
-  public BinaryIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testBinary() throws Exception {
     testJdbcTypeCompatibility("BINARY", new byte[0], new ByteArrayProvider());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
index 8a8d9bafc..35dfe2626 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
@@ -9,7 +9,6 @@
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -20,10 +19,6 @@ public class DateTimeIT extends AbstractDataTypeTest {
   private static final ZoneId TZ_BERLIN = ZoneId.of("Europe/Berlin");
   private static final ZoneId TZ_TOKYO = ZoneId.of("Asia/Tokyo");
 
-  public DateTimeIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Before
   public void setup() throws Exception {
     // Set to a random time zone not to interfere with any of the tests
@@ -101,7 +96,7 @@ public void testTimestampWithTimeZone() throws Exception {
         "2024-02-29 23:59:59.999999999 Z",
         new StringProvider(),
         new StringProvider());
-    expectArrowNotSupported("TIMESTAMP_TZ", "2023-02-29T23:59:59.999999999");
+    expectNotSupported("TIMESTAMP_TZ", "2023-02-29T23:59:59.999999999");
 
     // Numeric strings
     testJdbcTypeCompatibility(
@@ -345,7 +340,7 @@ public void testTimestampWithLocalTimeZone() throws Exception {
         "2024-02-29 15:59:59.999999999 -0800",
         new StringProvider(),
         new StringProvider());
-    expectArrowNotSupported("TIMESTAMP_LTZ", "2023-02-29T23:59:59.999999999");
+    expectNotSupported("TIMESTAMP_LTZ", "2023-02-29T23:59:59.999999999");
 
     // Numeric strings
     testJdbcTypeCompatibility(
@@ -587,7 +582,7 @@ public void testTimestampWithoutTimeZone() throws Exception {
         "2024-02-29 23:59:59.999999999 Z",
         new StringProvider(),
         new StringProvider());
-    expectArrowNotSupported("TIMESTAMP_NTZ", "2023-02-29T23:59:59.999999999");
+    expectNotSupported("TIMESTAMP_NTZ", "2023-02-29T23:59:59.999999999");
 
     // Numeric strings
     testJdbcTypeCompatibility(
@@ -927,7 +922,7 @@ public void testDate() throws Exception {
         "2024-02-29",
         new StringProvider(),
         new StringProvider());
-    expectArrowNotSupported("DATE", "2023-02-29T23:59:59.999999999");
+    expectNotSupported("DATE", "2023-02-29T23:59:59.999999999");
 
     // Test numeric strings
     testJdbcTypeCompatibility(
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
index 48936b950..6bc769a94 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/LogicalTypesIT.java
@@ -2,15 +2,10 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class LogicalTypesIT extends AbstractDataTypeTest {
 
-  public LogicalTypesIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testLogicalTypes() throws Exception {
     // Test booleans
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
index 22c59d3ea..fa01eebf0 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NullIT.java
@@ -1,15 +1,10 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
 import java.util.Arrays;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class NullIT extends AbstractDataTypeTest {
 
-  public NullIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testNullIngestion() throws Exception {
     for (String type :
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
index e9361e4a0..4e6ade712 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/NumericTypesIT.java
@@ -2,15 +2,10 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Test;
 
 public class NumericTypesIT extends AbstractDataTypeTest {
 
-  public NumericTypesIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testIntegers() throws Exception {
     // test bytes
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
index bf55b3396..49e994180 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
@@ -15,7 +15,6 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
-import net.snowflake.ingest.utils.Constants;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -25,10 +24,6 @@ public class SemiStructuredIT extends AbstractDataTypeTest {
   // server-side representation. Validation leaves a small buffer for this difference.
   private static final int MAX_ALLOWED_LENGTH = 16 * 1024 * 1024 - 64;
 
-  public SemiStructuredIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testVariant() throws Exception {
     // Test dates
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
index cfa59e0b1..63dc515f5 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/StringsIT.java
@@ -6,7 +6,6 @@
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
 import java.sql.SQLException;
-import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 import org.junit.Assert;
@@ -17,10 +16,6 @@ public class StringsIT extends AbstractDataTypeTest {
 
   private static final int MB_16 = 16 * 1024 * 1024;
 
-  public StringsIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testStrings() throws Exception {
     testJdbcTypeCompatibility("VARCHAR", "", new StringProvider());
@@ -36,7 +31,7 @@ public void testStrings() throws Exception {
     testJdbcTypeCompatibility("CHAR(5)", true, "true", new BooleanProvider(), new StringProvider());
     testJdbcTypeCompatibility(
         "CHAR(5)", false, "false", new BooleanProvider(), new StringProvider());
-    expectArrowNotSupported("CHAR(4)", false);
+    expectNotSupported("CHAR(4)", false);
 
     // test numbers
     testJdbcTypeCompatibility(
@@ -95,23 +90,23 @@ public void testMaxAllowedString() throws Exception {
     // 1-byte chars
     String maxString = buildString("a", MB_16);
     testIngestion("VARCHAR", maxString, new StringProvider());
-    expectArrowNotSupported("VARCHAR", maxString + "a");
+    expectNotSupported("VARCHAR", maxString + "a");
 
     // 2-byte chars
     maxString = buildString("š", MB_16 / 2);
     testIngestion("VARCHAR", maxString, new StringProvider());
 
-    expectArrowNotSupported("VARCHAR", maxString + "a");
+    expectNotSupported("VARCHAR", maxString + "a");
 
     // 3-byte chars
     maxString = buildString("❄", MB_16 / 3);
     testIngestion("VARCHAR", maxString, new StringProvider());
-    expectArrowNotSupported("VARCHAR", maxString + "aa");
+    expectNotSupported("VARCHAR", maxString + "aa");
 
     // 4-byte chars
     maxString = buildString("🍞", MB_16 / 4);
     testIngestion("VARCHAR", maxString, new StringProvider());
-    expectArrowNotSupported("VARCHAR", maxString + "a");
+    expectNotSupported("VARCHAR", maxString + "a");
   }
 
   @Test
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/it/ColumnNamesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/it/ColumnNamesIT.java
index fc2b220f4..254da0d52 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/it/ColumnNamesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/it/ColumnNamesIT.java
@@ -13,7 +13,6 @@
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.internal.datatypes.AbstractDataTypeTest;
-import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.SFException;
 import org.junit.Assert;
 import org.junit.Test;
@@ -21,10 +20,6 @@
 public class ColumnNamesIT extends AbstractDataTypeTest {
   private static final int INGEST_VALUE = 1;
 
-  public ColumnNamesIT(String name, Constants.BdecVersion bdecVersion) {
-    super(name, bdecVersion);
-  }
-
   @Test
   public void testColumnNamesSupport() throws Exception {
     // Test simple case

From 0c4da0c21cda4be9dcf7c2fe892ef3900e20876d Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Fri, 2 Jun 2023 14:25:32 -0700
Subject: [PATCH 239/356] SNOW-747848: Cleanup BG threads with exceptions
 (#479)

- Improve some comments
- Cleanup BG threads with exceptions in CTOR, reported by customer
---
 .../streaming/internal/FlushService.java      |  5 +--
 ...owflakeStreamingIngestChannelInternal.java |  4 ++-
 ...nowflakeStreamingIngestClientInternal.java | 33 ++++++++++++-------
 .../SnowflakeStreamingIngestClientTest.java   | 22 +++++++++++++
 4 files changed, 50 insertions(+), 14 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index ae4d04a23..59cb7194f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -601,7 +601,8 @@ String getFilePath(Calendar calendar, String clientPrefix) {
     int minute = calendar.get(Calendar.MINUTE);
     long time = TimeUnit.MILLISECONDS.toSeconds(calendar.getTimeInMillis());
     long threadId = Thread.currentThread().getId();
-    String fileName =
+    // Create the file short name, the clientPrefix contains the deployment id
+    String fileShortName =
         Long.toString(time, 36)
             + "_"
             + clientPrefix
@@ -611,7 +612,7 @@ String getFilePath(Calendar calendar, String clientPrefix) {
             + this.counter.getAndIncrement()
             + "."
             + BLOB_EXTENSION_TYPE;
-    return year + "/" + month + "/" + day + "/" + hour + "/" + minute + "/" + fileName;
+    return year + "/" + month + "/" + day + "/" + hour + "/" + minute + "/" + fileShortName;
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 136713324..b2d523a58 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -377,7 +377,9 @@ void collectRowSize(float rowSize) {
   }
 
   /**
-   * Get the latest committed offset token from Snowflake
+   * Get the latest committed offset token from Snowflake, an exception will be thrown if the
+   * channel becomes invalid due to errors and the channel needs to be reopened in order to return a
+   * valid offset token
    *
    * @return the latest committed offset token
    */
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index ec396c6d3..42d3d6709 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -99,8 +99,6 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
   // Name of the client
   private final String name;
 
-  private String accountName;
-
   // Snowflake role for the client to use
   private String role;
 
@@ -162,7 +160,7 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
     this.parameterProvider = new ParameterProvider(parameterOverrides, prop);
 
     this.name = name;
-    this.accountName = accountURL == null ? null : accountURL.getAccount();
+    String accountName = accountURL == null ? null : accountURL.getAccount();
     this.isTestMode = isTestMode;
     this.httpClient = httpClient == null ? HttpUtil.getHttpClient(accountName) : httpClient;
     this.channelCache = new ChannelCache<>();
@@ -191,7 +189,13 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
       this.setupMetricsForClient();
     }
 
-    this.flushService = new FlushService<>(this, this.channelCache, this.isTestMode);
+    try {
+      this.flushService = new FlushService<>(this, this.channelCache, this.isTestMode);
+    } catch (Exception e) {
+      // Need to clean up the resources before throwing any exceptions
+      cleanUpResources();
+      throw e;
+    }
 
     logger.logInfo(
         "Client created, name={}, account={}. isTestMode={}, parameters={}",
@@ -617,14 +621,8 @@ public void close() throws Exception {
     } catch (InterruptedException | ExecutionException e) {
       throw new SFException(e, ErrorCode.RESOURCE_CLEANUP_FAILURE, "client close");
     } finally {
-      if (this.telemetryWorker != null) {
-        this.telemetryWorker.shutdown();
-      }
       this.flushService.shutdown();
-      if (this.requestBuilder != null) {
-        this.requestBuilder.closeResources();
-      }
-      HttpUtil.shutdownHttpConnectionManagerDaemonThread();
+      cleanUpResources();
     }
   }
 
@@ -890,4 +888,17 @@ private void reportStreamingIngestTelemetryToSF() {
       telemetryService.reportCpuMemoryUsage(this.cpuHistogram);
     }
   }
+
+  /** Cleanup any resource during client closing or failures */
+  private void cleanUpResources() {
+    if (this.telemetryWorker != null) {
+      this.telemetryWorker.shutdown();
+    }
+    if (this.requestBuilder != null) {
+      this.requestBuilder.closeResources();
+    }
+    if (!this.isTestMode) {
+      HttpUtil.shutdownHttpConnectionManagerDaemonThread();
+    }
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index 244457ccf..a134e17ea 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -1213,4 +1213,26 @@ public void testVerifyChannelsAreFullyCommittedSuccess() throws Exception {
 
     client.close();
   }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testFlushServiceException() throws Exception {
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    RequestBuilder requestBuilder =
+        Mockito.spy(
+            new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair()));
+
+    // Set IO_TIME_CPU_RATIO to MAX_VALUE in order to generate an exception
+    Map<String, Object> parameterMap = new HashMap<>();
+    parameterMap.put(ParameterProvider.IO_TIME_CPU_RATIO, Integer.MAX_VALUE);
+
+    SnowflakeStreamingIngestClientInternal<?> client =
+        new SnowflakeStreamingIngestClientInternal<>(
+            "client",
+            new SnowflakeURL("snowflake.dev.local:8082"),
+            null,
+            httpClient,
+            true,
+            requestBuilder,
+            parameterMap);
+  }
 }

From e3413c186c5f80cb782a4a1b53831d152ecd8f06 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Tue, 6 Jun 2023 12:56:52 +0200
Subject: [PATCH 240/356] SNOW-830955 Use default SDK version in User-Agent
 (#513)

While constructing HTTP User-Agent header, take the SDK version from code instead from a file on classpath. It is less prone to accidental classpath overrides. The consequence of this is that we must now keep the DEFAULT_VERSION in sync with the project version defined in Maven, which we are doing already anyway (a new test has been added to verify this).
---
 .../ingest/connection/RequestBuilder.java     | 30 +------------------
 .../ingest/connection/UserAgentTest.java      | 20 +++++++++++++
 2 files changed, 21 insertions(+), 29 deletions(-)
 create mode 100644 src/test/java/net/snowflake/ingest/connection/UserAgentTest.java

diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 47e1574b5..824bac257 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -10,15 +10,12 @@
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import java.io.FileNotFoundException;
-import java.io.InputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.security.KeyPair;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Properties;
 import java.util.UUID;
 import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders;
 import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpGet;
@@ -28,7 +25,6 @@
 import net.snowflake.client.jdbc.internal.apache.http.entity.ContentType;
 import net.snowflake.client.jdbc.internal.apache.http.entity.StringEntity;
 import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
-import net.snowflake.ingest.SimpleIngestManager;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.SnowflakeURL;
@@ -118,10 +114,6 @@ public class RequestBuilder {
   // and object mapper for all marshalling and unmarshalling
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
-  private static final String RESOURCES_FILE = "project.properties";
-
-  private static final Properties PROPERTIES = loadProperties();
-
   private static final String USER_AGENT = getDefaultUserAgent();
 
   // Don't change!
@@ -298,25 +290,6 @@ public RequestBuilder(
         this.userAgentSuffix);
   }
 
-  private static Properties loadProperties() {
-    Properties properties = new Properties();
-    properties.put("version", DEFAULT_VERSION);
-
-    try (InputStream is =
-        SimpleIngestManager.class.getClassLoader().getResourceAsStream(RESOURCES_FILE)) {
-      if (is == null) {
-        throw new FileNotFoundException(RESOURCES_FILE);
-      }
-      properties.load(is);
-    } catch (Exception e) {
-      LOGGER.warn("Could not read version info, use default version " + DEFAULT_VERSION, e);
-      return properties;
-    }
-
-    LOGGER.info("Loaded project version " + properties.getProperty("version"));
-    return properties;
-  }
-
   /**
    * Creates a string for user agent which should always be present in all requests to Snowflake
    * (Snowpipe APIs)
@@ -328,8 +301,7 @@ private static Properties loadProperties() {
    * @return the default agent string
    */
   private static String getDefaultUserAgent() {
-    final String clientVersion = PROPERTIES.getProperty("version");
-    StringBuilder defaultUserAgent = new StringBuilder(CLIENT_NAME + "/" + clientVersion);
+    StringBuilder defaultUserAgent = new StringBuilder(CLIENT_NAME + "/" + DEFAULT_VERSION);
 
     final String osInformation =
         String.format(
diff --git a/src/test/java/net/snowflake/ingest/connection/UserAgentTest.java b/src/test/java/net/snowflake/ingest/connection/UserAgentTest.java
new file mode 100644
index 000000000..396dfa9ae
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/connection/UserAgentTest.java
@@ -0,0 +1,20 @@
+package net.snowflake.ingest.connection;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+import net.snowflake.ingest.SimpleIngestManager;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class UserAgentTest {
+  @Test
+  public void testDefaultSdkVersionMatchesProjectVersion() throws IOException {
+    Properties properties = new Properties();
+    try (InputStream is =
+        SimpleIngestManager.class.getClassLoader().getResourceAsStream("project.properties")) {
+      properties.load(is);
+      Assert.assertEquals(RequestBuilder.DEFAULT_VERSION, properties.getProperty("version"));
+    }
+  }
+}

From 85cfdca78a5809901b1f89d89ab2a5c616f89553 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Wed, 7 Jun 2023 14:17:49 +0200
Subject: [PATCH 241/356] NO-SNOW Exclude examples from code coverage scanning
 (#515)

---
 pom.xml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pom.xml b/pom.xml
index dd972a28c..f377edd83 100644
--- a/pom.xml
+++ b/pom.xml
@@ -532,6 +532,10 @@
           <version>${jacoco.version}</version>
           <configuration>
             <skip>${jacoco.skip.instrument}</skip>
+            <excludes>
+              <exclude>**/*SnowflakeStreamingIngestExample*</exclude>
+              <exclude>**/*SnowflakeIngestBasicExample*</exclude>
+            </excludes>
           </configuration>
           <executions>
             <execution>

From 271df95dc93f54081e97fba1c8613c5539e7db21 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Thu, 8 Jun 2023 13:34:09 +0200
Subject: [PATCH 242/356] SNOW-830862 Maven build improvements (#514)

---
 README.md                           |  13 +-
 linkage-checker-exclusion-rules.xml | 423 ++++++++++------------------
 pom.xml                             | 133 +++++----
 scripts/run_gh_actions.sh           |   2 +-
 4 files changed, 227 insertions(+), 344 deletions(-)

diff --git a/README.md b/README.md
index 8d5b455a0..2afb14e06 100644
--- a/README.md
+++ b/README.md
@@ -97,16 +97,9 @@ However, for general usage, pulling a pre-built jar from maven is
 recommended.
 
 If you would like to run SnowflakeIngestBasicExample.java or SnowflakeStreamingIngestExample.java in the example folder, 
-you would need to remove the following scope limits in pom.xml
-
-<pre>
-&lt;!-- Remove provided scope from slf4j-api --&gt;
-&lt;dependency&gt;
-    &lt;groupId&gt;org.slf4j&lt;/groupId&gt;
-    &lt;artifactId&gt;slf4j-api&lt;/artifactId&gt;
-    <s>&lt;scope&gt;provided&lt;/scope&gt;</s>
-&lt;/dependency&gt;
-</pre>
+please edit `pom.xml` and change the scope of the dependency `slf4j-simple` from `test` to `runtime` in order to enable
+console log output.
+
 
 # Testing (SimpleIngestIT Test)
 
diff --git a/linkage-checker-exclusion-rules.xml b/linkage-checker-exclusion-rules.xml
index 946b0b480..3544c6d48 100644
--- a/linkage-checker-exclusion-rules.xml
+++ b/linkage-checker-exclusion-rules.xml
@@ -1,278 +1,149 @@
 <LinkageCheckerFilter>
-  <LinkageError>
-    <Source><Package name="com.nimbusds.jose"/></Source>
-    <Reason>Optional</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="org.conscrypt"/></Target>
-    <Reason>Optional</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="com.google.crypto.tink.subtle"/></Target>
-    <Reason>Optional</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="com.sun.jna"/></Target>
-    <Reason>Optional</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="org.tukaani.xz"/></Target>
-    <Reason>Optional</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="org.apache.hadoop.mapred"/></Source>
-    <Reason>Conflict due to parquet-hadoop uses provided scope of hadoo-common</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="org.apache.hadoop.security.authentication"/></Target>
-    <Reason>We do not need hadoop authentication</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="org.apache.hadoop.util.PlatformName"/></Target>
-    <Reason>Relates to security/SSL pulled in by hadoop-auth, not used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="net.snowflake.ingest.internal.apache.hadoop.mapreduce"/></Target>
-    <Reason>There is no way we use mapreduce in this SDK</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="net.snowflake.ingest.internal.apache.hadoop.zookeeper"/></Target>
-    <Reason>There is no way we use Zookeeper in this SDK</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.client.jdbc.internal"/></Source>
-    <Reason>Unknown: Likely the referenced class is not used in the jdbc package</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.com.nimbusds.jose.crypto"/></Source>
-    <Reason>Not Used, appears to be Legacy</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.com.nimbusds.jose.jwk"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Class name="net.snowflake.ingest.internal.com.nimbusds.jose.util.X509CertChainUtils"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.apache.commons.configuration2"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.metrics2"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.fs"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.fs"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Class name="net.snowflake.ingest.internal.apache.hadoop.crypto.key.KeyShell "/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.security"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.ipc"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.net"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Class name="net.snowflake.ingest.internal.apache.conf.StorageSize"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Class name="net.snowflake.ingest.internal.apache.commons.text.lookup.DateStringLookup"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.apache.commons.lang3"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.classification"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.apache.yetus"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="net.snowflake.ingest.internal.apache.commons.lang3.time.FastDateFormat"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.io.nativeio"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.apache.commons.text"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="net.snowflake.ingest.internal.apache.commons.lang3.SystemUtils"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="net.snowflake.ingest.internal.apache.commons.lang3.StringUtils"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="net.snowflake.ingest.internal.apache.commons.lang3.builder.EqualsBuilder"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="net.snowflake.ingest.internal.apache.commons.lang3.builder.HashCodeBuilder"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Class name="javax.activation.MimetypesFileTypeMap"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="org.eclipse.jetty"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="net.snowflake.ingest.internal.apache.zookeeper"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="org.apache.zookeeper"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="net.snowflake.ingest.internal.fasterxml.jackson.jaxrs"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Class name="com.sun.jersey.json.impl.MoxyXmlStructure"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="com.sun.xml.bind.v2.runtime.output"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="com.sun.xml.bind.v2.runtime.unmarshaller"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Class name="net.snowflake.ingest.internal.apache.commons.logging.impl.LogKitLogger"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="org.joda.time.DateMidnight"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="org.joda.time.DateMidnight"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Class name="org.codehaus.jackson.map.ext.JodaSerializers"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Class name="org.codehaus.jackson.map.ext.JodaDeserializers"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="net.snowflake.ingest.internal.com.nimbusds.jwt.ReadOnlyJWTClaimsSet"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Class name="jcraft.jsch.jcraft.Compression"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="org.brotli.dec.BrotliInputStream"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Class name="com.ctc.wstx.msv.GenericMsvValidator"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.apache.commons.compress.harmony"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="net.snowflake.ingest.internal.apache.hadoop.mapred"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="net.snowflake.ingest.internal.apache.hadoop.security"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="net.snowflake.ingest.internal.net.objectweb"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="com.sun.msv.verifier.regexp.xmlschema"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="com.sun.javadoc.DocErrorReporter"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="com.sun.msv.reader.xmlschema.XMLSchemaReader"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="com.sun.msv.verifier.regexp.xmlschema.XSREDocDecl"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="com.sun.msv.reader.util.IgnoreController"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="com.jcraft.jzlib.ZStream"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="com.sun.msv.verifier.regexp.REDocumentDeclaration"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Class name="com.sun.msv.reader.trex.ng.RELAXNGReader"/></Target>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Package name="net.snowflake.ingest.internal.apache.hadoop.log"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Class name="org.apache.hadoop.log.LogLevel"/></Source>
-    <Reason>Not Used</Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Class name="org.apache.hadoop.log.metrics.EventCounter"/></Source>
-    <Reason></Reason>
-  </LinkageError>
-  <LinkageError>
-    <Source><Class name="org.apache.hadoop.log.metrics.Log4Json"/></Source>
-    <Reason></Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="org.apache.log4j"/></Target>
-    <Reason></Reason>
-  </LinkageError>
-  <LinkageError>
-    <Target><Package name="net.snowflake.ingest.internal.apache.log4j"/></Target>
-    <Reason></Reason>
-  </LinkageError>
+    <!-- Optional dependencies -->
+    <LinkageError>
+        <Source>
+            <Package name="com.nimbusds.jose"/>
+        </Source>
+        <Target>
+            <Package name="org.bouncycastle"/>
+        </Target>
+        <Reason>Bouncy castle is an optional dependency of nimbus-jose-jwt</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Source>
+            <Package name="com.nimbusds.jose"/>
+        </Source>
+        <Target>
+            <Package name="com.google.crypto.tink.subtle"/>
+        </Target>
+        <Reason>Google Crypto Tink is an optional dependency of nimbus-jose-jwt</Reason>
+    </LinkageError>
+
+    <!-- Hadoop-related linkage errors-->
+    <LinkageError>
+        <Source>
+            <Class name="org.apache.hadoop.http.HttpServer2"/>
+        </Source>
+        <Reason>Not used in the SDK</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Target>
+            <Class name="org.apache.hadoop.util.PlatformName"/>
+        </Target>
+        <Source>
+            <Class name="org.apache.hadoop.security.ssl.SSLFactory"/>
+        </Source>
+        <Reason>Relates to security/SSL pulled in by hadoop-auth, not used</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Target>
+            <Package name="org.apache.hadoop.security.authentication"/>
+        </Target>
+        <Reason>We do not need Hadoop authentication</Reason>
+    </LinkageError>
+
+    <!-- Snowflake JDBC linkage errors -->
+    <LinkageError>
+        <Source>
+            <Package name="net.snowflake.client.jdbc"/>
+        </Source>
+        <Reason>We don't explicitly exclude any dependency of the JDBC driver.
+            If some classes are missing, they know what are they doing.
+        </Reason>
+    </LinkageError>
+    <LinkageError>
+        <Target>
+            <Package name="com.sun.jna"/>
+        </Target>
+        <Source>
+            <Package name="net.snowflake.client.core"/>
+        </Source>
+        <Reason>Optional dependency of Snowflake JDBC driver that the SDK doesn't need</Reason>
+    </LinkageError>
+
+    <!-- Other linkage errors-->
+    <LinkageError>
+        <Target>
+            <Class name="org.eclipse.jetty.servlet.DefaultServlet"/>
+        </Target>
+    </LinkageError>
+
+    <LinkageError>
+        <Source>
+            <Package name="org.apache.hadoop.mapred"/>
+        </Source>
+        <Reason>The SDK is not using mapreduce</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Source>
+            <Class name="org.apache.hadoop.mapreduce.task.reduce.ShuffleClientMetrics"/>
+        </Source>
+        <Reason>The SDK is not using mapreduce</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Target>
+            <Package name="org.apache.hadoop.yarn"/>
+        </Target>
+        <Reason>The SDK is not using Hadoop YARN</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Target>
+            <Package name="org.tukaani.xz"/>
+        </Target>
+        <Reason>The SDK does not use XZ compression</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Target>
+            <Package name="org.apache.curator"/>
+        </Target>
+        <Reason>Client library for Apache ZooKeeper, not used in the SDK</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Target>
+            <Package name="com.jcraft.jsch"/>
+        </Target>
+        <Reason>SSH client, not used in the SDK</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Target>
+            <Package name="org.apache.avro"/>
+        </Target>
+        <Reason>The SDK is not using Avro</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Target>
+            <Package name="org.codehaus.jackson"/>
+        </Target>
+        <Reason>Not used, the SDK is using FasterXML Jackson</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Target>
+            <Package name="org.xbill.DNS"/>
+        </Target>
+        <Reason>Comes from dnsjava, which is not used in the SDK</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Target>
+            <Package name="org.apache.zookeeper"/>
+        </Target>
+        <Reason>Not used in the SDK</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Target>
+            <Package name="org.apache.log4j"/>
+        </Target>
+        <Reason>The SDK does not log with log4j</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Target>
+            <Package name="javax.ws.rs" />
+        </Target>
+        <Reason>The SDK does not use web services</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Target>
+            <Package name="org.eclipse.jetty.server" />
+        </Target>
+        <Reason>The SDK does not use Jetty web server</Reason>
+    </LinkageError>
 </LinkageCheckerFilter>
diff --git a/pom.xml b/pom.xml
index f377edd83..8ed318b1e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -296,7 +296,6 @@
         <artifactId>objenesis</artifactId>
         <version>${objenesis.version}</version>
       </dependency>
-      <!-- SLF4J api that a client can shim in later -->
       <dependency>
         <groupId>org.slf4j</groupId>
         <artifactId>slf4j-api</artifactId>
@@ -445,7 +444,6 @@
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
-      <scope>provided</scope>
     </dependency>
     <!-- JUnit so that we can make some basic unit tests -->
     <dependency>
@@ -487,6 +485,7 @@
       <version>2.0.2</version>
       <scope>test</scope>
     </dependency>
+    <!-- Change the 'test' scope to 'runtime' to enable console logging in examples -->
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-simple</artifactId>
@@ -513,6 +512,29 @@
             <skip>true</skip>
           </configuration>
         </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-enforcer-plugin</artifactId>
+          <version>3.0.0-M3</version>
+          <dependencies>
+            <dependency>
+              <groupId>com.google.cloud.tools</groupId>
+              <artifactId>linkage-checker-enforcer-rules</artifactId>
+              <version>1.5.12</version>
+            </dependency>
+            <dependency>
+              <groupId>org.codehaus.mojo</groupId>
+              <artifactId>extra-enforcer-rules</artifactId>
+              <version>1.3</version>
+              <exclusions>
+                <exclusion>
+                  <groupId>org.eclipse.aether</groupId>
+                  <artifactId>aether-util</artifactId>
+                </exclusion>
+              </exclusions>
+            </dependency>
+          </dependencies>
+        </plugin>
 
         <!-- Need to define the maven-failsafe-plugin here -->
         <!-- http://maven.apache.org/surefire/maven-failsafe-plugin/plugin-info.html -->
@@ -633,25 +655,6 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-enforcer-plugin</artifactId>
-        <version>3.0.0-M3</version>
-        <dependencies>
-          <dependency>
-            <groupId>com.google.cloud.tools</groupId>
-            <artifactId>linkage-checker-enforcer-rules</artifactId>
-            <version>1.5.12</version>
-          </dependency>
-          <dependency>
-            <groupId>org.codehaus.mojo</groupId>
-            <artifactId>extra-enforcer-rules</artifactId>
-            <version>1.3</version>
-            <exclusions>
-              <exclusion>
-                <groupId>org.eclipse.aether</groupId>
-                <artifactId>aether-util</artifactId>
-              </exclusion>
-            </exclusions>
-          </dependency>
-        </dependencies>
         <executions>
           <execution>
             <id>enforce-best-practices</id>
@@ -684,28 +687,6 @@
               </rules>
             </configuration>
           </execution>
-          <execution>
-            <id>enforce-linkage-checker</id>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <phase>verify</phase>
-            <configuration>
-              <rules>
-                <LinkageCheckerRule implementation="com.google.cloud.tools.dependencies.enforcer.LinkageCheckerRule">
-                  <reportOnlyReachable>true</reportOnlyReachable>
-                  <exclusionFile>${basedir}/linkage-checker-exclusion-rules.xml</exclusionFile>
-                  <level>WARN</level>
-                </LinkageCheckerRule>
-                <bannedDependencies>
-                  <excludes>
-                    <!-- JUnit 4 is not allowed on the classpath to prevent its accidental use -->
-                    <!-- <exclude>junit:junit</exclude> -->
-                  </excludes>
-                </bannedDependencies>
-              </rules>
-            </configuration>
-          </execution>
         </executions>
       </plugin>
       <plugin>
@@ -771,14 +752,15 @@
           <failOnBlacklist>true</failOnBlacklist>
           <licenseMerges>
             <licenseMerge>Apache License 2.0
-              |The Apache License, Version 2.0
-              |The Apache Software License, Version 2.0
-              |Apache-2.0
-              |Apache License, Version 2.0
-              |Apache 2.0
-              |Apache License V2.0</licenseMerge>
+                            |The Apache License, Version 2.0
+                            |The Apache Software License, Version 2.0
+                            |Apache-2.0
+                            |Apache License, Version 2.0
+                            |Apache 2.0
+                            |Apache License V2.0</licenseMerge>
             <licenseMerge>BSD 2-Clause License
-              |The BSD License</licenseMerge>
+                            |The BSD License</licenseMerge>
+            <licenseMerge>The MIT License|MIT License</licenseMerge>
           </licenseMerges>
         </configuration>
         <executions>
@@ -796,17 +778,53 @@
 
   <profiles>
     <profile>
-      <id>checkShadedContent</id>
+      <id>checkLinkageErrors</id>
       <activation>
-        <os>
-          <!-- Shaded JAR content checking is a shell script, do not run it on Windows -->
-          <family>!Windows</family>
-        </os>
         <property>
-          <!-- Do not check shaded JAR content if shading has been disabled -->
-          <name>!not-shadeDep</name>
+          <!-- Only check linkage errors for unshaded builds -->
+          <name>not-shadeDep</name>
         </property>
       </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-enforcer-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>enforce-linkage-checker</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <phase>verify</phase>
+                <configuration>
+                  <rules>
+                    <LinkageCheckerRule implementation="com.google.cloud.tools.dependencies.enforcer.LinkageCheckerRule">
+                      <reportOnlyReachable>true</reportOnlyReachable>
+                      <exclusionFile>${basedir}/linkage-checker-exclusion-rules.xml</exclusionFile>
+                      <level>WARN</level>
+                    </LinkageCheckerRule>
+                    <bannedDependencies>
+                      <excludes>
+                        <!-- JUnit 4 is not allowed on the classpath to prevent its accidental use -->
+                        <!-- <exclude>junit:junit</exclude> -->
+                      </excludes>
+                    </bannedDependencies>
+                  </rules>
+                </configuration>
+              </execution>
+
+            </executions>
+          </plugin>
+
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <id>checkShadedContent</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
       <build>
         <plugins>
           <plugin>
@@ -846,6 +864,7 @@
               <artifactSet>
                 <excludes>
                   <exclude>net.snowflake:snowflake-jdbc</exclude>
+                  <exclude>org.slf4j:slf4j-api</exclude>
                 </excludes>
               </artifactSet>
               <relocations>
diff --git a/scripts/run_gh_actions.sh b/scripts/run_gh_actions.sh
index 452d67682..49895010b 100755
--- a/scripts/run_gh_actions.sh
+++ b/scripts/run_gh_actions.sh
@@ -3,7 +3,7 @@
 set -o pipefail
 
 # Build and install shaded JAR first. check_content.sh runs here.
-mvn install -DskipTests=true --batch-mode --show-version
+mvn install -PcheckShadedContent -DskipTests=true --batch-mode --show-version
 
 PARAMS=()
 PARAMS+=("-DghActionsIT")

From 9c84fc27a1ec87516582bc6f42336620b25819c8 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Thu, 8 Jun 2023 15:52:38 +0200
Subject: [PATCH 243/356] SNOW-768197 Attribution notice for shaded
 dependencies (#517)

---
 pom.xml                                       |  88 +++++++-
 scripts/process_licenses.py                   | 144 +++++++++++++
 .../third-party-licenses/Apache2.0.txt        | 201 ------------------
 .../third-party-licenses/BSD-2-Clause.txt     |  24 ---
 .../third-party-licenses/BSD-3-Clause.txt     |  28 ---
 .../META-INF/third-party-licenses/EDL1.0.txt  |  12 --
 .../META-INF/third-party-licenses/Go.txt      |  27 ---
 .../META-INF/third-party-licenses/MIT.txt     |  21 --
 8 files changed, 227 insertions(+), 318 deletions(-)
 create mode 100644 scripts/process_licenses.py
 delete mode 100644 src/main/resources/META-INF/third-party-licenses/Apache2.0.txt
 delete mode 100644 src/main/resources/META-INF/third-party-licenses/BSD-2-Clause.txt
 delete mode 100644 src/main/resources/META-INF/third-party-licenses/BSD-3-Clause.txt
 delete mode 100644 src/main/resources/META-INF/third-party-licenses/EDL1.0.txt
 delete mode 100644 src/main/resources/META-INF/third-party-licenses/Go.txt
 delete mode 100644 src/main/resources/META-INF/third-party-licenses/MIT.txt

diff --git a/pom.xml b/pom.xml
index 8ed318b1e..052876710 100644
--- a/pom.xml
+++ b/pom.xml
@@ -48,6 +48,10 @@
     <hadoop.version>3.3.5</hadoop.version>
     <jacoco.skip.instrument>true</jacoco.skip.instrument>
     <jacoco.version>0.8.5</jacoco.version>
+    <license.processing.dependencyJarsDir>${project.build.directory}/dependency-jars</license.processing.dependencyJarsDir>
+    <license.processing.dependencyListFile>${project.build.directory}/dependency-list.txt</license.processing.dependencyListFile>
+    <license.processing.resourcesRoot>${project.build.directory}/generated-licenses-info</license.processing.resourcesRoot>
+    <license.processing.targetDir>${license.processing.resourcesRoot}/META-INF/third-party-licenses</license.processing.targetDir>
     <maven.compiler.source>1.8</maven.compiler.source>
     <maven.compiler.target>1.8</maven.compiler.target>
     <net.minidev.version>2.4.9</net.minidev.version>
@@ -501,6 +505,9 @@
         <filtering>true</filtering>
         <directory>src/main/resources</directory>
       </resource>
+      <resource>
+        <directory>${license.processing.resourcesRoot}</directory>
+      </resource>
     </resources>
 
     <!-- disable default maven deploy plugin since we are using gpg:sign-and-deploy-file -->
@@ -735,6 +742,10 @@
         <version>2.0.1</version>
         <configuration>
           <errorRemedy>failFast</errorRemedy>
+          <!--
+          The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please
+          check your dependencies", verify the conditions of the license and add the reference to it here.
+          -->
           <includedLicenses>
             <includedLicense>Apache License 2.0</includedLicense>
             <includedLicense>BSD 2-Clause License</includedLicense>
@@ -743,11 +754,6 @@
             <includedLicense>EDL 1.0</includedLicense>
             <includedLicense>The Go license</includedLicense>
           </includedLicenses>
-          <!--
-          The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please
-          check your dependencies", verify the conditions of the license, put the text of the license into third-party-licenses/
-          directory and add it to this list. You can also run "mvn license:third-party-report" to get an HTML license report.
-          -->
           <excludedScopes>test,provided,system</excludedScopes>
           <failOnBlacklist>true</failOnBlacklist>
           <licenseMerges>
@@ -855,6 +861,52 @@
       </activation>
       <build>
         <plugins>
+          <!--
+          Copy all project dependencies to target/dependency-jars. License processing Python script will look here for
+          license files of SDK dependencies.
+          -->
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-dependency-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>copy-dependencies</id>
+                <goals>
+                  <goal>copy-dependencies</goal>
+                </goals>
+                <phase>generate-resources</phase>
+                <configuration>
+                  <outputDirectory>${license.processing.dependencyJarsDir}</outputDirectory>
+                  <overWriteReleases>false</overWriteReleases>
+                  <overWriteSnapshots>false</overWriteSnapshots>
+                  <overWriteIfNewer>true</overWriteIfNewer>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+
+          <!--
+          Compile the list of SDK dependencies in 'compile' and 'runtime' scopes.
+          This list is an entry point for the license processing python script.
+          -->
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-dependency-plugin</artifactId>
+            <version>3.6.0</version>
+            <configuration>
+              <includeScope>runtime</includeScope>
+              <outputFile>${license.processing.dependencyListFile}</outputFile>
+            </configuration>
+            <executions>
+              <execution>
+                <goals>
+                  <goal>list</goal>
+                </goals>
+                <phase>generate-resources</phase>
+              </execution>
+            </executions>
+          </plugin>
+
           <!-- Relocate all dependencies to internal to solve dependency conflict problem -->
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
@@ -1010,6 +1062,32 @@
               </execution>
             </executions>
           </plugin>
+          <!--
+          Plugin executes license processing Python script, which copies third party license files into the directory
+          target/generated-licenses-info/META-INF/third-party-licenses, which is then included in the shaded JAR. 
+          -->
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>process-third-party-licenses</id>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <phase>generate-resources</phase>
+                <configuration>
+                  <executable>python3</executable>
+                  <arguments>
+                    <argument>${project.basedir}/scripts/process_licenses.py</argument>
+                    <argument>${license.processing.dependencyListFile}</argument>
+                    <argument>${license.processing.dependencyJarsDir}</argument>
+                    <argument>${license.processing.targetDir}</argument>
+                  </arguments>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
         </plugins>
       </build>
     </profile>
diff --git a/scripts/process_licenses.py b/scripts/process_licenses.py
new file mode 100644
index 000000000..69fafefed
--- /dev/null
+++ b/scripts/process_licenses.py
@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+
+# This script processes licenses of 3rd party dependencies and stores them in the JAR. The rules are:
+# 1. Dependencies, which contains a license file should be put into the shaded JAR as-is.
+# 2. Dependencies, which do not contain a license file should be mentioned in the file ADDITIONAL_LICENCES, together with the name of its license.
+#
+#
+# The script accepts the following arguments:
+# * DEPENDENCY_LIST_FILE_PATH
+#     * Can be obtained by running mvn dependency:list -DincludeScope=runtime -DoutputFile=target/dependency_list.txt
+# * DEPENDENCIES_DIR
+#     * Directory containging the JAR files of all SDK dependencies. Automatically generated by `mvn clean package` in target/dependency-jars
+# * TARGET_DIR
+#     * Where to save all output, should be target/generated-sources/META-INF/third-party-licenses
+#
+#
+# Useful mvn commands:
+# * mvn clean license:add-third-party
+#     * Generate dependency report; useful to find out licenses for dependencies that don't ship with a license file
+# * mvn dependency:list -DincludeScope=runtime -DoutputFile=target/dependency_list.txt
+#     * Used as input of this script (DEPENDENCY_LIST_FILE_PATH)
+import sys
+from pathlib import Path
+from zipfile import ZipFile
+
+# License name constants
+APACHE_LICENSE = "Apache License 2.0"
+BSD_2_CLAUSE_LICENSE = "2-Clause BSD License"
+BSD_3_CLAUSE_LICENSE = "3-Clause BSD License"
+EDL_10_LICENSE = "EDL 1.0"
+MIT_LICENSE = "The MIT License"
+GO_LICENSE = "The Go license"
+
+# The SDK does not need to include licenses of dependencies, which aren't shaded
+IGNORED_DEPENDENCIES = {"net.snowflake:snowflake-jdbc", "org.slf4j:slf4j-api"}
+
+# List of dependencies, which don't ship with a license file.
+# Only add a new record here after verifying that the dependency JAR does not contain a license!
+ADDITIONAL_LICENSES_MAP = {
+    "com.google.code.findbugs:jsr305": APACHE_LICENSE,
+    "io.dropwizard.metrics:metrics-core": APACHE_LICENSE,
+    "io.dropwizard.metrics:metrics-jmx": APACHE_LICENSE,
+    "io.dropwizard.metrics:metrics-jvm": APACHE_LICENSE,
+    "com.google.guava:guava": APACHE_LICENSE,
+    "com.google.guava:failureaccess": APACHE_LICENSE,
+    "com.google.guava:listenablefuture": APACHE_LICENSE,
+    "com.google.errorprone:error_prone_annotations": APACHE_LICENSE,
+    "com.google.j2objc:j2objc-annotations": APACHE_LICENSE,
+    "com.nimbusds:nimbus-jose-jwt": APACHE_LICENSE,
+    "com.github.stephenc.jcip:jcip-annotations": APACHE_LICENSE,
+    "io.netty:netty-common": APACHE_LICENSE,
+    "com.google.re2j:re2j": GO_LICENSE,
+    "com.google.protobuf:protobuf-java": BSD_3_CLAUSE_LICENSE,
+    "com.google.code.gson:gson": APACHE_LICENSE,
+    "org.xerial.snappy:snappy-java": APACHE_LICENSE,
+    "org.apache.parquet:parquet-common": APACHE_LICENSE,
+    "org.apache.parquet:parquet-format-structures": APACHE_LICENSE,
+    "com.github.luben:zstd-jni": BSD_2_CLAUSE_LICENSE,
+}
+
+
+def parse_cmdline_args():
+    if len(sys.argv) != 4:
+        raise Exception("usage: process_licenses.py DEPENDENCY_LIST_FILE_PATH DEPENDENCIES_DIR TARGET_DIR")
+    dependency_list_file_path = Path(sys.argv[1]).absolute()
+    dependencies_dir_path = Path(sys.argv[2]).absolute()
+    target_dir = Path(sys.argv[3]).absolute()
+
+    if not dependency_list_file_path.exists() or not dependency_list_file_path.is_file():
+        raise Exception(f"File {dependency_list_file_path} does not exist")
+
+    if not dependencies_dir_path.exists() or not dependencies_dir_path.is_dir():
+        raise Exception(f"Directory {dependencies_dir_path} does not exist")
+    return dependency_list_file_path, dependencies_dir_path, target_dir
+
+
+def main():
+    dependency_list_path, dependency_jars_path, target_dir = parse_cmdline_args()
+
+    dependency_count = 0
+    dependency_with_license_count = 0
+    dependency_without_license_count = 0
+    dependency_ignored_count = 0
+
+    missing_licenses_str = ""
+
+    target_dir.mkdir(parents=True, exist_ok=True)
+
+    with open(dependency_list_path, "r") as dependency_file_handle:
+        for line in dependency_file_handle.readlines():
+            line = line.strip()
+            if line == "" or line == "The following files have been resolved:":
+                continue
+            dependency_count += 1
+
+            # Line is a string like: "commons-codec:commons-codec:jar:1.15:compile -- module org.apache.commons.codec [auto]"
+            artifact_details = line.split()[0]
+            group_id, artifact_id, _, version, scope = artifact_details.split(":")
+            current_jar = Path(dependency_jars_path, f"{artifact_id}-{version}.jar")
+            if not current_jar.exists() and current_jar.is_file():
+                raise Exception(f"Expected JAR file does not exist: {current_jar}")
+            current_jar_as_zip = ZipFile(current_jar)
+
+            dependency_lookup_key = f"{group_id}:{artifact_id}"
+            if dependency_lookup_key in IGNORED_DEPENDENCIES:
+                dependency_ignored_count += 1
+                continue
+
+            license_found = False
+            for zip_info in current_jar_as_zip.infolist():
+                if zip_info.is_dir():
+                    continue
+                if zip_info.filename in ("META-INF/LICENSE.txt", "META-INF/LICENSE", "META-INF/LICENSE.md"):
+                    license_found = True
+                    dependency_with_license_count += 1
+                    # Extract license to the target directory
+                    zip_info.filename = f"LICENSE_{group_id}__{artifact_id}"
+                    current_jar_as_zip.extract(zip_info, target_dir)
+                    break
+                if "license" in zip_info.filename.lower():  # Log potential license matches
+                    print(f"Potential license match: {current_jar} {zip_info}")
+
+            if not license_found:
+                print(f"License not found {current_jar}; using value from ADDITIONAL_LICENSES_MAP")
+                license_name = ADDITIONAL_LICENSES_MAP.get(dependency_lookup_key)
+                if license_name:
+                    dependency_without_license_count += 1
+                    missing_licenses_str += f"{dependency_lookup_key}: {license_name}\n"
+                else:
+                    raise Exception(f"The dependency {dependency_lookup_key} does not ship a license file, but neither is it not defined in ADDITIONAL_LICENSES_MAP")
+
+    with open(Path(target_dir, "ADDITIONAL_LICENCES"), "w") as additional_licenses_handle:
+        additional_licenses_handle.write(missing_licenses_str)
+
+    if dependency_count < 30:
+        raise Exception(f"Suspiciously low number of dependency JARs detected in {dependency_jars_path}: {dependency_count}")
+    print("License generation finished")
+    print(f"\tTotal dependencies: {dependency_count}")
+    print(f"\tTotal dependencies (with license): {dependency_with_license_count}")
+    print(f"\tTotal dependencies (without license): {dependency_without_license_count}")
+    print(f"\tIgnored dependencies: {dependency_ignored_count}")
+
+if __name__ == "__main__":
+    main()
diff --git a/src/main/resources/META-INF/third-party-licenses/Apache2.0.txt b/src/main/resources/META-INF/third-party-licenses/Apache2.0.txt
deleted file mode 100644
index f49a4e16e..000000000
--- a/src/main/resources/META-INF/third-party-licenses/Apache2.0.txt
+++ /dev/null
@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
\ No newline at end of file
diff --git a/src/main/resources/META-INF/third-party-licenses/BSD-2-Clause.txt b/src/main/resources/META-INF/third-party-licenses/BSD-2-Clause.txt
deleted file mode 100644
index 805473dfd..000000000
--- a/src/main/resources/META-INF/third-party-licenses/BSD-2-Clause.txt
+++ /dev/null
@@ -1,24 +0,0 @@
-BSD 2-Clause License
-
-Copyright (c) [year], [fullname]
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this
-   list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice,
-   this list of conditions and the following disclaimer in the documentation
-   and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/src/main/resources/META-INF/third-party-licenses/BSD-3-Clause.txt b/src/main/resources/META-INF/third-party-licenses/BSD-3-Clause.txt
deleted file mode 100644
index ae5698f13..000000000
--- a/src/main/resources/META-INF/third-party-licenses/BSD-3-Clause.txt
+++ /dev/null
@@ -1,28 +0,0 @@
-BSD 3-Clause License
-
-Copyright (c) [year], [fullname]
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this
-   list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice,
-   this list of conditions and the following disclaimer in the documentation
-   and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-   contributors may be used to endorse or promote products derived from
-   this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/src/main/resources/META-INF/third-party-licenses/EDL1.0.txt b/src/main/resources/META-INF/third-party-licenses/EDL1.0.txt
deleted file mode 100644
index a4a267ef0..000000000
--- a/src/main/resources/META-INF/third-party-licenses/EDL1.0.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-Eclipse Distribution License - v 1.0
-
-Copyright (c) 2007, Eclipse Foundation, Inc. and its licensors.
-
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
-
-Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
-Neither the name of the Eclipse Foundation, Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/src/main/resources/META-INF/third-party-licenses/Go.txt b/src/main/resources/META-INF/third-party-licenses/Go.txt
deleted file mode 100644
index 6a66aea5e..000000000
--- a/src/main/resources/META-INF/third-party-licenses/Go.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-Copyright (c) 2009 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/src/main/resources/META-INF/third-party-licenses/MIT.txt b/src/main/resources/META-INF/third-party-licenses/MIT.txt
deleted file mode 100644
index 63b4b681c..000000000
--- a/src/main/resources/META-INF/third-party-licenses/MIT.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) [year] [fullname]
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
\ No newline at end of file

From d23a8a95699a4dca53520677bd93a83100393fae Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 8 Jun 2023 14:21:45 -0700
Subject: [PATCH 244/356] NO-SNOW: Limit the chunk size to avoid potential OOM
 issue in the Parquet Reader (#516)

Given that we have some limitations in the current Parquet Reader, this PR contains the change to limit the chunk size to avoid any potential OOM risks. It's currently set to 128MB uncompressed based on the reader expert and we also add the support to override it if needed to avoid any potential regressions.
---
 .../streaming/internal/FlushService.java      | 36 ++++++++++---
 ...owflakeStreamingIngestChannelInternal.java |  6 +--
 .../ingest/utils/ParameterProvider.java       | 18 ++++++-
 .../streaming/internal/FlushServiceTest.java  | 50 ++++++++++++++++++-
 .../internal/ParameterProviderTest.java       |  8 +--
 .../streaming/internal/RowBufferTest.java     |  4 +-
 6 files changed, 102 insertions(+), 20 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 59cb7194f..86358b24d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -259,7 +259,7 @@ CompletableFuture<Void> flush(boolean isForce) {
 
     if (isForce
         || (!DISABLE_BACKGROUND_FLUSH
-            && !this.isTestMode
+            && !isTestMode()
             && (this.isNeedFlush
                 || timeDiffMillis
                     >= this.owningClient.getParameterProvider().getBufferFlushIntervalInMs()))) {
@@ -370,21 +370,28 @@ void distributeFlushTasks() {
 
         if (!channelsDataPerTable.isEmpty()) {
           int idx = 0;
+          float totalBufferSizePerTableInBytes = 0F;
           while (idx < channelsDataPerTable.size()) {
             ChannelData<T> channelData = channelsDataPerTable.get(idx);
             // Stop processing the rest of channels when needed
             if (idx > 0
                 && shouldStopProcessing(
-                    totalBufferSizeInBytes, channelData, channelsDataPerTable.get(idx - 1))) {
+                    totalBufferSizeInBytes,
+                    totalBufferSizePerTableInBytes,
+                    channelData,
+                    channelsDataPerTable.get(idx - 1))) {
               leftoverChannelsDataPerTable.addAll(
                   channelsDataPerTable.subList(idx, channelsDataPerTable.size()));
               logger.logInfo(
-                  "Creation of another blob is needed because of blob size limit or different"
-                      + " encryption ids or different schema, client={}, table={},  size={},"
-                      + " encryptionId1={}, encryptionId2={}, schema1={}, schema2={}",
+                  "Creation of another blob is needed because of blob/chunk size limit or"
+                      + " different encryption ids or different schema, client={}, table={},"
+                      + " fileSize={}, chunkSize={}, nextChannelSize={}, encryptionId1={},"
+                      + " encryptionId2={}, schema1={}, schema2={}",
                   this.owningClient.getName(),
                   channelData.getChannelContext().getTableName(),
-                  totalBufferSizeInBytes + channelData.getBufferSize(),
+                  totalBufferSizeInBytes,
+                  totalBufferSizePerTableInBytes,
+                  channelData.getBufferSize(),
                   channelData.getChannelContext().getEncryptionKeyId(),
                   channelsDataPerTable.get(idx - 1).getChannelContext().getEncryptionKeyId(),
                   channelData.getColumnEps().keySet(),
@@ -392,6 +399,7 @@ && shouldStopProcessing(
               break;
             }
             totalBufferSizeInBytes += channelData.getBufferSize();
+            totalBufferSizePerTableInBytes += channelData.getBufferSize();
             idx++;
           }
           // Add processed channels to the current blob, stop if we need to create a new blob
@@ -473,15 +481,22 @@ && shouldStopProcessing(
    * Check whether we should stop merging more channels into the same chunk, we need to stop in a
    * few cases:
    *
-   * <p>When the size is larger than a certain threshold
+   * <p>When the file size is larger than a certain threshold
+   *
+   * <p>When the chunk size is larger than a certain threshold
    *
    * <p>When the encryption key ids are not the same
    *
    * <p>When the schemas are not the same
    */
   private boolean shouldStopProcessing(
-      float totalBufferSizeInBytes, ChannelData<T> current, ChannelData<T> prev) {
+      float totalBufferSizeInBytes,
+      float totalBufferSizePerTableInBytes,
+      ChannelData<T> current,
+      ChannelData<T> prev) {
     return totalBufferSizeInBytes + current.getBufferSize() > MAX_BLOB_SIZE_IN_BYTES
+        || totalBufferSizePerTableInBytes + current.getBufferSize()
+            > this.owningClient.getParameterProvider().getMaxChunkSizeInBytes()
         || !Objects.equals(
             current.getChannelContext().getEncryptionKeyId(),
             prev.getChannelContext().getEncryptionKeyId())
@@ -658,4 +673,9 @@ boolean throttleDueToQueuedFlushTasks() {
     }
     return throttleOnQueuedTasks;
   }
+
+  /** Get whether we're running under test mode */
+  boolean isTestMode() {
+    return this.isTestMode;
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index b2d523a58..f5dfb73a6 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -126,8 +126,8 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
                 ? owningClient.getParameterProvider().getEnableParquetInternalBuffering()
                 : ParameterProvider.ENABLE_PARQUET_INTERNAL_BUFFERING_DEFAULT,
             owningClient != null
-                ? owningClient.getParameterProvider().getMaxChunkSizeInBytes()
-                : ParameterProvider.MAX_CHUNK_SIZE_IN_BYTES_DEFAULT);
+                ? owningClient.getParameterProvider().getMaxChannelSizeInBytes()
+                : ParameterProvider.MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT);
     logger.logInfo(
         "Channel={} created for table={}",
         this.channelFlushContext.getName(),
@@ -362,7 +362,7 @@ public InsertValidationResponse insertRows(
     // TODO: Checking table/chunk level size reduces throughput a lot, we may want to check it only
     // if a large number of rows are inserted
     if (this.rowBuffer.getSize()
-        >= this.owningClient.getParameterProvider().getMaxChunkSizeInBytes()) {
+        >= this.owningClient.getParameterProvider().getMaxChannelSizeInBytes()) {
       this.owningClient.setNeedFlush();
     }
 
diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
index 7faea2859..451682f52 100644
--- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
+++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -25,6 +25,8 @@ public class ParameterProvider {
   public static final String MAX_MEMORY_LIMIT_IN_BYTES = "MAX_MEMORY_LIMIT_IN_BYTES".toLowerCase();
   public static final String ENABLE_PARQUET_INTERNAL_BUFFERING =
       "ENABLE_PARQUET_INTERNAL_BUFFERING".toLowerCase();
+  // This should not be needed once we have the ability to track size at table/chunk level
+  public static final String MAX_CHANNEL_SIZE_IN_BYTES = "MAX_CHANNEL_SIZE_IN_BYTES".toLowerCase();
   public static final String MAX_CHUNK_SIZE_IN_BYTES = "MAX_CHUNK_SIZE_IN_BYTES".toLowerCase();
 
   // Default values
@@ -39,7 +41,8 @@ public class ParameterProvider {
   public static final int IO_TIME_CPU_RATIO_DEFAULT = 2;
   public static final int BLOB_UPLOAD_MAX_RETRY_COUNT_DEFAULT = 24;
   public static final long MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT = -1L;
-  public static final long MAX_CHUNK_SIZE_IN_BYTES_DEFAULT = 32000000L;
+  public static final long MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT = 32000000L;
+  public static final long MAX_CHUNK_SIZE_IN_BYTES_DEFAULT = 128000000L;
 
   /* Parameter that enables using internal Parquet buffers for buffering of rows before serializing.
   It reduces memory consumption compared to using Java Objects for buffering.*/
@@ -138,6 +141,9 @@ private void setParameterMap(Map<String, Object> parameterOverrides, Properties
         parameterOverrides,
         props);
 
+    this.updateValue(
+        MAX_CHANNEL_SIZE_IN_BYTES, MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT, parameterOverrides, props);
+
     this.updateValue(
         MAX_CHUNK_SIZE_IN_BYTES, MAX_CHUNK_SIZE_IN_BYTES_DEFAULT, parameterOverrides, props);
   }
@@ -265,7 +271,15 @@ public boolean getEnableParquetInternalBuffering() {
     return (val instanceof String) ? Boolean.parseBoolean(val.toString()) : (boolean) val;
   }
 
-  /** @return The max chunk size in bytes */
+  /** @return The max channel size in bytes */
+  public long getMaxChannelSizeInBytes() {
+    Object val =
+        this.parameterMap.getOrDefault(
+            MAX_CHANNEL_SIZE_IN_BYTES, MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT);
+    return (val instanceof String) ? Long.parseLong(val.toString()) : (long) val;
+  }
+
+  /** @return The max chunk size in bytes that could avoid OOM at server side */
   public long getMaxChunkSizeInBytes() {
     Object val =
         this.parameterMap.getOrDefault(MAX_CHUNK_SIZE_IN_BYTES, MAX_CHUNK_SIZE_IN_BYTES_DEFAULT);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index fc049c297..38e6ac8ca 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -7,6 +7,7 @@
 import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
 import static net.snowflake.ingest.utils.Constants.BLOB_TAG_SIZE_IN_BYTES;
 import static net.snowflake.ingest.utils.Constants.BLOB_VERSION_SIZE_IN_BYTES;
+import static net.snowflake.ingest.utils.ParameterProvider.MAX_CHUNK_SIZE_IN_BYTES_DEFAULT;
 
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
@@ -92,7 +93,7 @@ private abstract static class TestContext<T> implements AutoCloseable {
       channelCache = new ChannelCache<>();
       Mockito.when(client.getChannelCache()).thenReturn(channelCache);
       registerService = Mockito.spy(new RegisterService(client, client.isTestMode()));
-      flushService = Mockito.spy(new FlushService<>(client, channelCache, stage, false));
+      flushService = Mockito.spy(new FlushService<>(client, channelCache, stage, true));
     }
 
     ChannelData<T> flushChannel(String name) {
@@ -360,6 +361,18 @@ private static ColumnMetadata createTestTextColumn(String name) {
     return colChar;
   }
 
+  private static ColumnMetadata createLargeTestTextColumn(String name) {
+    ColumnMetadata colChar = new ColumnMetadata();
+    colChar.setName(name);
+    colChar.setPhysicalType("LOB");
+    colChar.setNullable(true);
+    colChar.setLogicalType("TEXT");
+    colChar.setByteLength(14000000);
+    colChar.setLength(11000000);
+    colChar.setScale(0);
+    return colChar;
+  }
+
   @Test
   public void testGetFilePath() {
     TestContext<?> testContext = testContextFactory.create();
@@ -398,6 +411,7 @@ public void testGetFilePath() {
   public void testFlush() throws Exception {
     TestContext<?> testContext = testContextFactory.create();
     FlushService<?> flushService = testContext.flushService;
+    Mockito.when(flushService.isTestMode()).thenReturn(false);
 
     // Nothing to flush
     flushService.flush(false).get();
@@ -505,6 +519,40 @@ public void testBlobSplitDueToDifferentSchema() throws Exception {
     Mockito.verify(flushService, Mockito.atLeast(2)).buildAndUpload(Mockito.any(), Mockito.any());
   }
 
+  @Test
+  public void testBlobSplitDueToChunkSizeLimit() throws Exception {
+    TestContext<?> testContext = testContextFactory.create();
+    SnowflakeStreamingIngestChannelInternal<?> channel1 = addChannel1(testContext);
+    SnowflakeStreamingIngestChannelInternal<?> channel2 = addChannel2(testContext);
+    String colName1 = "testBlobSplitDueToChunkSizeLimit1";
+    String colName2 = "testBlobSplitDueToChunkSizeLimit2";
+    int rowSize = 10000000;
+    String largeData = new String(new char[rowSize]);
+
+    List<ColumnMetadata> schema =
+        Arrays.asList(createTestIntegerColumn(colName1), createLargeTestTextColumn(colName2));
+    channel1.getRowBuffer().setupSchema(schema);
+    channel2.getRowBuffer().setupSchema(schema);
+
+    RowSetBuilder builder = RowSetBuilder.newBuilder();
+    RowSetBuilder.newBuilder().addColumn(colName1, 11).addColumn(colName2, largeData);
+
+    for (int idx = 0; idx <= MAX_CHUNK_SIZE_IN_BYTES_DEFAULT / (2 * rowSize); idx++) {
+      builder.addColumn(colName1, 11).addColumn(colName2, largeData).newRow();
+    }
+
+    List<Map<String, Object>> rows = builder.build();
+
+    channel1.insertRows(rows, "offset1");
+    channel2.insertRows(rows, "offset2");
+
+    FlushService<?> flushService = testContext.flushService;
+
+    // Force = true flushes
+    flushService.flush(true).get();
+    Mockito.verify(flushService, Mockito.times(2)).buildAndUpload(Mockito.any(), Mockito.any());
+  }
+
   @Test
   public void testBuildAndUpload() throws Exception {
     long expectedBuildLatencyMs = 100;
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
index 1fe034635..dcf4037c6 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
@@ -21,7 +21,7 @@ public void withValuesSet() {
     parameterMap.put(ParameterProvider.IO_TIME_CPU_RATIO, 10);
     parameterMap.put(ParameterProvider.BLOB_UPLOAD_MAX_RETRY_COUNT, 100);
     parameterMap.put(ParameterProvider.MAX_MEMORY_LIMIT_IN_BYTES, 1000L);
-    parameterMap.put(ParameterProvider.MAX_CHUNK_SIZE_IN_BYTES, 1000000L);
+    parameterMap.put(ParameterProvider.MAX_CHANNEL_SIZE_IN_BYTES, 1000000L);
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
 
     Assert.assertEquals(3L, parameterProvider.getBufferFlushIntervalInMs());
@@ -32,7 +32,7 @@ public void withValuesSet() {
     Assert.assertEquals(10, parameterProvider.getIOTimeCpuRatio());
     Assert.assertEquals(100, parameterProvider.getBlobUploadMaxRetryCount());
     Assert.assertEquals(1000L, parameterProvider.getMaxMemoryLimitInBytes());
-    Assert.assertEquals(1000000L, parameterProvider.getMaxChunkSizeInBytes());
+    Assert.assertEquals(1000000L, parameterProvider.getMaxChannelSizeInBytes());
   }
 
   @Test
@@ -120,7 +120,7 @@ public void withDefaultValues() {
         ParameterProvider.MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT,
         parameterProvider.getMaxMemoryLimitInBytes());
     Assert.assertEquals(
-        ParameterProvider.MAX_CHUNK_SIZE_IN_BYTES_DEFAULT,
-        parameterProvider.getMaxChunkSizeInBytes());
+        ParameterProvider.MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT,
+        parameterProvider.getMaxChannelSizeInBytes());
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index b6e035c90..b60fbf9f9 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -1,7 +1,7 @@
 package net.snowflake.ingest.streaming.internal;
 
 import static java.time.ZoneOffset.UTC;
-import static net.snowflake.ingest.utils.ParameterProvider.MAX_CHUNK_SIZE_IN_BYTES_DEFAULT;
+import static net.snowflake.ingest.utils.ParameterProvider.MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -115,7 +115,7 @@ private AbstractRowBuffer<?> createTestBuffer(OpenChannelRequest.OnErrorOption o
         rs -> {},
         initialState,
         enableParquetMemoryOptimization,
-        MAX_CHUNK_SIZE_IN_BYTES_DEFAULT);
+        MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT);
   }
 
   @Test

From 335959a59a534fe52693c5d8833f9bddd1d9e712 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Fri, 9 Jun 2023 18:40:39 +0200
Subject: [PATCH 245/356] NO-SNOW Enforce allowed DATE and TIMESTAMP range
 (#520)

---
 .../internal/DataValidationUtil.java          | 12 +++++
 .../internal/DataValidationUtilTest.java      | 23 ++++++++-
 .../internal/datatypes/DateTimeIT.java        | 49 ++++++++++++++++---
 3 files changed, 77 insertions(+), 7 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index d04f9d3fc..1bdfc2095 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -454,6 +454,11 @@ static TimestampWrapper validateAndParseTimestamp(
     if (trimTimezone) {
       offsetDateTime = offsetDateTime.withOffsetSameLocal(ZoneOffset.UTC);
     }
+    if (offsetDateTime.getYear() < 1 || offsetDateTime.getYear() > 9999) {
+      throw new SFException(
+          ErrorCode.INVALID_VALUE_ROW,
+          "Timestamp out of representable inclusive range of years between 1 and 9999");
+    }
     return new TimestampWrapper(offsetDateTime, scale);
   }
 
@@ -579,6 +584,13 @@ static BigDecimal validateAndParseBigDecimal(
   static int validateAndParseDate(String columnName, Object input, long insertRowIndex) {
     OffsetDateTime offsetDateTime =
         inputToOffsetDateTime(columnName, "DATE", input, ZoneOffset.UTC, insertRowIndex);
+
+    if (offsetDateTime.getYear() < -9999 || offsetDateTime.getYear() > 9999) {
+      throw new SFException(
+          ErrorCode.INVALID_VALUE_ROW,
+          "Date out of representable inclusive range of years between -9999 and 9999");
+    }
+
     return Math.toIntExact(offsetDateTime.toLocalDate().toEpochDay());
   }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 6f3dd4558..f6f9e36af 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -118,6 +118,14 @@ public void testValidateAndParseDate() {
     // Time input is not supported
     expectError(ErrorCode.INVALID_VALUE_ROW, () -> validateAndParseDate("COL", "20:57:01", 0));
 
+    // Test values out of range
+    expectError(
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseDate("COL", LocalDateTime.of(10000, 2, 2, 2, 2), 0));
+    expectError(
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseDate("COL", LocalDateTime.of(-10000, 2, 2, 2, 2), 0));
+
     // Test forbidden values
     expectError(ErrorCode.INVALID_FORMAT_ROW, () -> validateAndParseDate("COL", new Object(), 0));
     expectError(
@@ -262,6 +270,19 @@ public void testValidateAndParseTimestamp() {
         ErrorCode.INVALID_VALUE_ROW,
         () -> validateAndParseTimestamp("COL", "20:57:01", 3, UTC, false, 0));
 
+    // Test values out of range
+    expectError(
+        ErrorCode.INVALID_VALUE_ROW,
+        () ->
+            validateAndParseTimestamp(
+                "COL", LocalDateTime.of(10000, 2, 2, 2, 2), 3, UTC, false, 0));
+    expectError(
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseTimestamp("COL", LocalDateTime.of(0, 2, 2, 2, 2), 3, UTC, false, 0));
+    expectError(
+        ErrorCode.INVALID_VALUE_ROW,
+        () -> validateAndParseTimestamp("COL", LocalDateTime.of(-1, 2, 2, 2, 2), 3, UTC, false, 0));
+
     // Test forbidden values
     expectError(
         ErrorCode.INVALID_FORMAT_ROW,
@@ -845,7 +866,7 @@ public void testValidateAndParseBinary() throws DecoderException {
         Hex.decodeHex("1234567890abcdef"), // pragma: allowlist secret NOT A SECRET
         validateAndParseBinary(
             "COL",
-            "1234567890abcdef",
+            "1234567890abcdef", // pragma: allowlist secret NOT A SECRET
             Optional.empty(),
             0)); // pragma: allowlist secret NOT A SECRET
     assertArrayEquals(
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
index 35dfe2626..9bb8f349a 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
@@ -947,19 +947,56 @@ public void testDate() throws Exception {
 
   @Test
   public void testOldTimestamps() throws Exception {
-    testIngestion("DATE", "0001-12-31", "0001-12-31", new StringProvider());
-    testIngestion(
+    // DATE
+    testJdbcTypeCompatibility("DATE", "0001-12-31", new StringProvider());
+    testJdbcTypeCompatibility("DATE", "0000-01-01", new StringProvider());
+    testJdbcTypeCompatibility("DATE", "-0001-01-01", new StringProvider());
+    testJdbcTypeCompatibility("DATE", "-9999-01-01", new StringProvider());
+
+    // TIMESTAMP_NTZ
+    testJdbcTypeCompatibility(
         "TIMESTAMP_NTZ",
         "0001-12-31T11:11:11",
         "0001-12-31 11:11:11.000000000 Z",
+        new StringProvider(),
         new StringProvider());
-    // TODO uncomment once SNOW-727474 is resolved
-    //        testIngestion("TIMESTAMP_LTZ", "0001-12-31T11:11:11", "0001-12-31 03:11:11.000000000
-    // -0800", new StringProvider());
-    testIngestion(
+
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "0001-01-01T00:00:00",
+        "0001-01-01 00:00:00.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+
+    // TIMESTAMP_TZ
+    testJdbcTypeCompatibility(
         "TIMESTAMP_TZ",
         "0001-12-31T11:11:11+03:00",
         "0001-12-31 11:11:11.000000000 +0300",
+        new StringProvider(),
+        new StringProvider());
+
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_TZ",
+        "0001-01-01T00:00:00+03:00",
+        "0001-01-01 00:00:00.000000000 +0300",
+        new StringProvider(),
+        new StringProvider());
+
+    // TIMESTAMP_LTZ
+    conn.createStatement()
+        .execute("alter session set timezone = 'UTC';"); // workaround for SNOW-727474
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "0001-12-31T11:11:11+00:00",
+        "0001-12-31 11:11:11.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_LTZ",
+        "0001-01-01T00:00:00+00:00",
+        "0001-01-01 00:00:00.000000000 Z",
+        new StringProvider(),
         new StringProvider());
   }
 

From b52c4c36ed81453b2f12874b3f2cb03968026fc6 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Fri, 9 Jun 2023 21:04:58 +0200
Subject: [PATCH 246/356] SNOW-720420 Enforce per-row size limits (#519)

---
 .../streaming/internal/AbstractRowBuffer.java |  6 +-
 .../streaming/internal/ParquetRowBuffer.java  | 15 +++-
 ...owflakeStreamingIngestChannelInternal.java |  5 +-
 .../net/snowflake/ingest/utils/ErrorCode.java |  3 +-
 .../ingest/utils/ParameterProvider.java       | 10 +++
 .../ingest/ingest_error_messages.properties   |  1 +
 .../streaming/internal/RowBufferTest.java     |  4 +-
 .../SnowflakeStreamingIngestChannelTest.java  | 82 +++++++++++++++++++
 8 files changed, 120 insertions(+), 6 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index c5d0bddac..e9addc4a9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -529,7 +529,8 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelRuntimeState,
       boolean enableParquetMemoryOptimization,
-      long maxChunkSizeInBytes) {
+      long maxChunkSizeInBytes,
+      long maxRowSizeInBytes) {
     switch (bdecVersion) {
       case THREE:
         //noinspection unchecked
@@ -541,7 +542,8 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
                 rowSizeMetric,
                 channelRuntimeState,
                 enableParquetMemoryOptimization,
-                maxChunkSizeInBytes);
+                maxChunkSizeInBytes,
+                maxRowSizeInBytes);
       default:
         throw new SFException(
             ErrorCode.INTERNAL_ERROR, "Unsupported BDEC format version: " + bdecVersion);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 03b1c1762..baf7fa3bc 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -53,6 +53,7 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
   private MessageType schema;
   private final boolean enableParquetInternalBuffering;
   private final long maxChunkSizeInBytes;
+  private final long maxAllowedRowSizeInBytes;
 
   /** Construct a ParquetRowBuffer object. */
   ParquetRowBuffer(
@@ -62,7 +63,8 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelRuntimeState,
       boolean enableParquetInternalBuffering,
-      long maxChunkSizeInBytes) {
+      long maxChunkSizeInBytes,
+      long maxAllowedRowSizeInBytes) {
     super(
         onErrorOption,
         defaultTimezone,
@@ -76,6 +78,7 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
     this.channelName = fullyQualifiedChannelName;
     this.enableParquetInternalBuffering = enableParquetInternalBuffering;
     this.maxChunkSizeInBytes = maxChunkSizeInBytes;
+    this.maxAllowedRowSizeInBytes = maxAllowedRowSizeInBytes;
   }
 
   @Override
@@ -202,6 +205,16 @@ private float addRow(
       indexedRow[colIndex] = valueWithSize.getValue();
       size += valueWithSize.getSize();
     }
+
+    long rowSizeRoundedUp = Double.valueOf(Math.ceil(size)).longValue();
+
+    if (rowSizeRoundedUp > maxAllowedRowSizeInBytes) {
+      throw new SFException(
+          ErrorCode.MAX_ROW_SIZE_EXCEEDED,
+          String.format(
+              "rowSizeInBytes=%.3f maxAllowedRowSizeInBytes=%d", size, maxAllowedRowSizeInBytes));
+    }
+
     out.accept(Arrays.asList(indexedRow));
 
     // All input values passed validation, iterate over the columns again and combine their existing
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index f5dfb73a6..5ddc654ab 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -127,7 +127,10 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
                 : ParameterProvider.ENABLE_PARQUET_INTERNAL_BUFFERING_DEFAULT,
             owningClient != null
                 ? owningClient.getParameterProvider().getMaxChannelSizeInBytes()
-                : ParameterProvider.MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT);
+                : ParameterProvider.MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT,
+            owningClient != null
+                ? owningClient.getParameterProvider().getMaxAllowedRowSizeInBytes()
+                : ParameterProvider.MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT);
     logger.logInfo(
         "Channel={} created for table={}",
         this.channelFlushContext.getName(),
diff --git a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
index bacb3d556..92968fb80 100644
--- a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
+++ b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
@@ -35,7 +35,8 @@ public enum ErrorCode {
   ENCRYPTION_FAILURE("0027"),
   CHANNEL_STATUS_INVALID("0028"),
   UNSUPPORTED_DATA_TYPE("0029"),
-  INVALID_VALUE_ROW("0030");
+  INVALID_VALUE_ROW("0030"),
+  MAX_ROW_SIZE_EXCEEDED("0031");
 
   public static final String errorMessageResource = "net.snowflake.ingest.ingest_error_messages";
 
diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
index 451682f52..7e4c3d042 100644
--- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
+++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -28,6 +28,8 @@ public class ParameterProvider {
   // This should not be needed once we have the ability to track size at table/chunk level
   public static final String MAX_CHANNEL_SIZE_IN_BYTES = "MAX_CHANNEL_SIZE_IN_BYTES".toLowerCase();
   public static final String MAX_CHUNK_SIZE_IN_BYTES = "MAX_CHUNK_SIZE_IN_BYTES".toLowerCase();
+  public static final String MAX_ALLOWED_ROW_SIZE_IN_BYTES =
+      "MAX_ALLOWED_ROW_SIZE_IN_BYTES".toLowerCase();
 
   // Default values
   public static final long BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT = 1000;
@@ -43,6 +45,7 @@ public class ParameterProvider {
   public static final long MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT = -1L;
   public static final long MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT = 32000000L;
   public static final long MAX_CHUNK_SIZE_IN_BYTES_DEFAULT = 128000000L;
+  public static final long MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT = 64 * 1024 * 1024; // 64 MB
 
   /* Parameter that enables using internal Parquet buffers for buffering of rows before serializing.
   It reduces memory consumption compared to using Java Objects for buffering.*/
@@ -286,6 +289,13 @@ public long getMaxChunkSizeInBytes() {
     return (val instanceof String) ? Long.parseLong(val.toString()) : (long) val;
   }
 
+  public long getMaxAllowedRowSizeInBytes() {
+    Object val =
+        this.parameterMap.getOrDefault(
+            MAX_ALLOWED_ROW_SIZE_IN_BYTES, MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT);
+    return (val instanceof String) ? Long.parseLong(val.toString()) : (long) val;
+  }
+
   @Override
   public String toString() {
     return "ParameterProvider{" + "parameterMap=" + parameterMap + '}';
diff --git a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
index 5aaeadfba..160748106 100644
--- a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
+++ b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
@@ -33,3 +33,4 @@
 0028=Get channel status indicates Channel {0} is invalid with status code {1}, please reopen the channel.
 0029=Data type not supported: {0}
 0030=The given row cannot be converted to the internal format due to invalid value: {0}
+0031=The given row exceeds the maximum allowed row size {0}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index b60fbf9f9..321d76c93 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -1,6 +1,7 @@
 package net.snowflake.ingest.streaming.internal;
 
 import static java.time.ZoneOffset.UTC;
+import static net.snowflake.ingest.utils.ParameterProvider.MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT;
 import static net.snowflake.ingest.utils.ParameterProvider.MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT;
 
 import java.math.BigDecimal;
@@ -115,7 +116,8 @@ private AbstractRowBuffer<?> createTestBuffer(OpenChannelRequest.OnErrorOption o
         rs -> {},
         initialState,
         enableParquetMemoryOptimization,
-        MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT);
+        MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT,
+        MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT);
   }
 
   @Test
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 23c95d792..62f2efdce 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -14,11 +14,14 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import net.snowflake.client.core.SFSessionProperty;
 import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils;
 import net.snowflake.client.jdbc.internal.apache.http.HttpEntity;
@@ -549,6 +552,85 @@ public void testInsertRow() {
     Assert.assertTrue(insertEndTimeInMs >= data.getMinMaxInsertTimeInMs().getSecond());
   }
 
+  @Test
+  public void testInsertTooLargeRow() {
+    byte[] byteArrayOneMb = new byte[1024 * 1024];
+
+    List<ColumnMetadata> schema =
+        IntStream.range(0, 64)
+            .mapToObj(
+                rowId -> {
+                  ColumnMetadata col = new ColumnMetadata();
+                  col.setName("COL" + rowId);
+                  col.setPhysicalType("LOB");
+                  col.setNullable(false);
+                  col.setLogicalType("BINARY");
+                  col.setLength(8388608);
+                  col.setByteLength(8388608);
+                  return col;
+                })
+            .collect(Collectors.toList());
+
+    String expectedMessage =
+        "The given row exceeds the maximum allowed row size rowSizeInBytes=67109128.000"
+            + " maxAllowedRowSizeInBytes=67108864";
+
+    Map<String, Object> row = new HashMap<>();
+    schema.forEach(x -> row.put(x.getName(), byteArrayOneMb));
+
+    SnowflakeStreamingIngestClientInternal<?> client;
+    client = new SnowflakeStreamingIngestClientInternal<ParquetChunkData>("test_client");
+
+    // Test channel with on error CONTINUE
+    SnowflakeStreamingIngestChannelInternal<?> channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
+            "channel",
+            "db",
+            "schema",
+            "table",
+            "0",
+            0L,
+            0L,
+            client,
+            "key",
+            1234L,
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
+    channel.setupSchema(schema);
+
+    InsertValidationResponse insertValidationResponse = channel.insertRow(row, "token-1");
+    Assert.assertEquals(1, insertValidationResponse.getErrorRowCount());
+    SFException thrownException = insertValidationResponse.getInsertErrors().get(0).getException();
+    Assert.assertEquals(
+        ErrorCode.MAX_ROW_SIZE_EXCEEDED.getMessageCode(), thrownException.getVendorCode());
+    Assert.assertEquals(expectedMessage, thrownException.getMessage());
+
+    // Test channel with on error ABORT
+    channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
+            "channel",
+            "db",
+            "schema",
+            "table",
+            "0",
+            0L,
+            0L,
+            client,
+            "key",
+            1234L,
+            OpenChannelRequest.OnErrorOption.ABORT,
+            UTC);
+    channel.setupSchema(schema);
+
+    try {
+      channel.insertRow(row, "token-1");
+      Assert.fail("Insert row shouldn't have succeeded");
+    } catch (SFException e) {
+      Assert.assertEquals(ErrorCode.MAX_ROW_SIZE_EXCEEDED.getMessageCode(), e.getVendorCode());
+      Assert.assertEquals(expectedMessage, e.getMessage());
+    }
+  }
+
   @Test
   public void testInsertRowThrottling() {
     final long maxMemory = 1000000L;

From 8860d1998870d6b5a37417f7737f9823cb65a474 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Fri, 9 Jun 2023 22:21:39 +0200
Subject: [PATCH 247/356] NO-SNOW Mention minimal JDBC driver version 3.13.30
 (#523)

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2afb14e06..97e1bca92 100644
--- a/README.md
+++ b/README.md
@@ -13,6 +13,8 @@ authentication. Currently, we support ingestion through the following APIs:
 
 # Prerequisites
 
+**If your project depends on the Snowflake JDBC driver, as well, please make sure the JDBC driver version is 3.13.30 or newer.**
+
 ## Java 8+
 
 The Snowflake Ingest Service SDK can only be used with Java 8 or higher.
@@ -118,4 +120,4 @@ console log output.
 We use [Google Java format](https://github.com/google/google-java-format) to format the code. To format all files, run:
 ```bash
 ./format.sh
-````
\ No newline at end of file
+```

From fec56872cb9a10482c4edd9b016c51f1fea84958 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Fri, 9 Jun 2023 16:35:36 -0700
Subject: [PATCH 248/356] V2.0.0 release (#522)

Release 2.0.0, which will hopefully be the first stable release for internal GA
---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 052876710..4f2ca2ce9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>1.1.5-SNAPSHOT</version>
+  <version>2.0.0</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 824bac257..a053047bf 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -119,7 +119,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "1.1.5-SNAPSHOT";
+  public static final String DEFAULT_VERSION = "2.0.0";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 6bd0e852c00aa09ba9b35e46597bb8acfcbea459 Mon Sep 17 00:00:00 2001
From: Xin Huang <xin.huang@snowflake.com>
Date: Mon, 12 Jun 2023 09:04:15 -0700
Subject: [PATCH 249/356] NO-SNOW Exclude additional example from code coverage
 (#524)

---
 pom.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pom.xml b/pom.xml
index 4f2ca2ce9..021808e0b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -564,6 +564,7 @@
             <excludes>
               <exclude>**/*SnowflakeStreamingIngestExample*</exclude>
               <exclude>**/*SnowflakeIngestBasicExample*</exclude>
+              <exclude>**/*IngestExampleHelper*</exclude>
             </excludes>
           </configuration>
           <executions>

From e9b7c882aba89e99c0c62533f1d1b4998d634562 Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Tue, 13 Jun 2023 22:12:23 +0200
Subject: [PATCH 250/356] @snow SNOW-837436 Streaming ingest: get status code
 message in channel status response (#525)

Add status code message as part of the channel status response
---
 .../internal/ChannelRegisterStatus.java       |  10 +
 ...nowflakeStreamingIngestClientInternal.java |   4 +-
 .../internal/StreamingIngestResponseCode.java | 189 ++++++++++++++++++
 .../StreamingIngestResponseCodeTest.java      |  32 +++
 4 files changed, 234 insertions(+), 1 deletion(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestResponseCode.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestResponseCodeTest.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelRegisterStatus.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelRegisterStatus.java
index 3eaa13c64..5bdc791ec 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelRegisterStatus.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelRegisterStatus.java
@@ -12,6 +12,7 @@
  */
 class ChannelRegisterStatus {
   private Long statusCode;
+  private String message;
   private String channelName;
   private Long channelSequencer;
 
@@ -24,6 +25,15 @@ Long getStatusCode() {
     return this.statusCode;
   }
 
+  @JsonProperty("message")
+  public String getMessage() {
+    return message == null ? StreamingIngestResponseCode.getMessageByCode(statusCode) : message;
+  }
+
+  public void setMessage(String message) {
+    this.message = message;
+  }
+
   @JsonProperty("channel")
   void setChannelName(String channelName) {
     this.channelName = channelName;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 42d3d6709..3d478624b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -496,10 +496,12 @@ void registerBlobs(List<BlobMetadata> blobs, final int executionCount) {
                                               String.format(
                                                   "Channel has been invalidated because of failure"
                                                       + " response, name=%s, channel_sequencer=%d,"
-                                                      + " status_code=%d, executionCount=%d",
+                                                      + " status_code=%d,  message=%s,"
+                                                      + " executionCount=%d",
                                                   channelStatus.getChannelName(),
                                                   channelStatus.getChannelSequencer(),
                                                   channelStatus.getStatusCode(),
+                                                  channelStatus.getMessage(),
                                                   executionCount);
                                           logger.logWarn(errorMessage);
                                           if (getTelemetryService() != null) {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestResponseCode.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestResponseCode.java
new file mode 100644
index 000000000..19d957aa4
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestResponseCode.java
@@ -0,0 +1,189 @@
+/*
+ * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import com.google.common.annotations.VisibleForTesting;
+
+public enum StreamingIngestResponseCode {
+  SUCCESS(0, "Success"),
+  ERR_TABLE_DOES_NOT_EXIST_NOT_AUTHORIZED(
+      4, "The supplied table does not exist or is not authorized"),
+  ERR_CHANNEL_LIMIT_EXCEEDED_FOR_TABLE(
+      5,
+      "Channel limit exceeded for the given table, please contact Snowflake support to raise this"
+          + " limit."),
+  ERR_TABLE_OR_COLUMN_TYPE_NOT_SUPPORTED(
+      6,
+      "Snowpipe Streaming is not supported on the type of table resolved, or the table contains"
+          + " columns that are either AUTOINCREMENT or IDENTITY columns or a column with a default"
+          + " value"),
+  ERR_ENQUEUE_TABLE_CHUNK_QUEUE_FULL(
+      7,
+      "The table has exceeded its limit of outstanding registration requests, please wait before"
+          + " calling the insertRow or insertRows API"),
+  ERR_DATABASE_DOES_NOT_EXIST_OR_NOT_AUTHORIZED(
+      8, "The supplied database does not exist or is not authorized"),
+  ERR_SCHEMA_DOES_NOT_EXIST_OR_NOT_AUTHORIZED(
+      9, "The supplied schema does not exist or is not authorized"),
+  ERR_GENERAL_EXCEPTION_RETRY_REQUEST(
+      10, "Snowflake experienced a transient exception, please retry the request"),
+  ERR_MALFORMED_REQUEST_MISSING_BLOBS_IN_REQUEST(
+      11,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_MISSING_CHUNKS_FOR_BLOB_IN_REQUEST(
+      12,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_MISSING_CHANNELS_FOR_CHUNK_FOR_BLOB_IN_REQUEST(
+      13,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_MISSING_BLOB_PATH(
+      14,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_MISSING_DATABASE_IN_REQUEST(
+      15,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_MISSING_SCHEMA_IN_REQUEST(
+      16,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_MISSING_TABLE_IN_REQUEST(
+      17,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_MISSING_CHANNEL_NAME_IN_REQUEST(
+      18,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_CHANNEL_NO_LONGER_EXISTS(
+      19,
+      "The requested channel no longer exists, most likely due to inactivity. Please re-open the"
+          + " channel"),
+  ERR_CHANNEL_HAS_INVALID_CLIENT_SEQUENCER(
+      20,
+      "The channel is owned by another writer at this time. Either re-open the channel to gain"
+          + " exclusive write ownership or close the channel and let the other writer continue"),
+  ERR_CHANNEL_HAS_INVALID_ROW_SEQUENCER(
+      21, "The client provided an out-of-order message, please reopen the channel"),
+  ERR_OTHER_CHANNEL_IN_CHUNK_HAS_ISSUE(
+      22,
+      "Another channel managed by this Client had an issue which is preventing the current channel"
+          + " from ingesting data. Please re-open the channel"),
+  ERR_MALFORMED_REQUEST_DUPLICATE_CHANNEL_IN_CHUNK(
+      23,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_MISSING_CLIENT_SEQUENCER_IN_REQUEST(
+      24,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_CHANNEL_DOES_NOT_EXIST_OR_IS_NOT_AUTHORIZED(
+      25, "The channel does not exist or is not authorized"),
+  ROW_SEQUENCER_IS_COMMITTED(26, "The requested row sequencer is committed"),
+  ROW_SEQUENCER_IS_NOT_COMMITTED(27, "The requested row sequencer is not committed"),
+
+  ERR_MALFORMED_REQUEST_MISSING_CHUNK_MD5_IN_REQUEST(
+      28,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_MISSING_EP_INFO_IN_REQUEST(
+      29,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_INVALID_CHUNK_LENGTH(
+      30,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_INVALID_ROW_COUNT_IN_EP_INFO(
+      31,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_INVALID_COLUMN_IN_EP_INFO(
+      32,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_INVALID_EP_INFO_GENERIC(
+      33,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_INVALID_BLOB_NAME(
+      34,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+
+  ERR_CHANNEL_MUST_BE_REOPENED(36, "The channel must be reopened"),
+  ERR_MALFORMED_REQUEST_MISSING_ROLE_IN_REQUEST(
+      37,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_BLOB_HAS_WRONG_FORMAT_OR_EXTENSION(
+      38,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_BLOB_MISSING_MD5(
+      39,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MISSING_COLUMN_IN_EP_INFO(
+      40,
+      "A schema change occurred on the table, please re-open the channel and supply the missing"
+          + " data"),
+  ERR_DUPLICATE_BLOB_IN_REQUEST(41, "Duplicated blob in request"),
+  ERR_UNSUPPORTED_BLOB_FILE_VERSION(42, "Unsupported blob file version"),
+  ERR_OUTDATED_CLIENT_SDK_VERSION(
+      43, "Outdated Client SDK. Please update your SDK to the latest version"),
+  ERR_MALFORMED_REQUEST_UNDEFINED_USER_AGENT_HEADER(
+      44,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_MALFORMED_REQUEST_MALFORMED_USER_AGENT_HEADER(
+      45,
+      "The SDK generated a malformed request. Please contact Snowflake support for further"
+          + " assistance"),
+  ERR_INTERLEAVING_NOT_SUPPORTED_AT_THIS_TIME(
+      46, "Interleaving among tables is not supported at this time"),
+  ERR_RESOLVED_TABLE_IS_READ_ONLY(47, "Table is read-only"),
+  ERR_MALFORMED_REQUEST_INVALID_VAL_IN_EP_INFO(
+      48, "The request contains invalid column metadata, please contact Snowflake support"),
+  ERR_INGESTION_ON_TABLE_NOT_ALLOWED(
+      49,
+      "Ingestion into this table is not allowed at this time, please contact Snowflake support");
+
+  public static final String UNKNOWN_STATUS_MESSAGE = "unknown";
+
+  private final long statusCode;
+
+  private final String message;
+
+  StreamingIngestResponseCode(int statusCode, String message) {
+    this.statusCode = statusCode;
+    this.message = message;
+  }
+
+  @VisibleForTesting
+  public long getStatusCode() {
+    return statusCode;
+  }
+
+  @VisibleForTesting
+  public String getMessage() {
+    return message;
+  }
+
+  public static String getMessageByCode(Long statusCode) {
+    if (statusCode != null) {
+      for (StreamingIngestResponseCode code : StreamingIngestResponseCode.values()) {
+        if (code.statusCode == statusCode) {
+          return code.message;
+        }
+      }
+    }
+    return UNKNOWN_STATUS_MESSAGE;
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestResponseCodeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestResponseCodeTest.java
new file mode 100644
index 000000000..f209ef9e9
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestResponseCodeTest.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class StreamingIngestResponseCodeTest {
+  @Test
+  public void testGetMessageNullCode() {
+    Assert.assertEquals(
+        StreamingIngestResponseCode.UNKNOWN_STATUS_MESSAGE,
+        StreamingIngestResponseCode.getMessageByCode(null));
+  }
+
+  @Test
+  public void testGetMessageUnknownCode() {
+    Assert.assertEquals(
+        StreamingIngestResponseCode.UNKNOWN_STATUS_MESSAGE,
+        StreamingIngestResponseCode.getMessageByCode(-1L));
+  }
+
+  @Test
+  public void testGetMessageKnownCodes() {
+    for (StreamingIngestResponseCode code : StreamingIngestResponseCode.values()) {
+      Assert.assertEquals(
+          code.getMessage(), StreamingIngestResponseCode.getMessageByCode(code.getStatusCode()));
+    }
+  }
+}

From f560e920005969218bd2c5be2a4630a4b221c654 Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Wed, 14 Jun 2023 07:58:03 +0000
Subject: [PATCH 251/356] @snow SNOW-837436 Streaming ingest: dedup var for
 support status code messages

---
 .../internal/StreamingIngestResponseCode.java | 95 ++++++-------------
 1 file changed, 28 insertions(+), 67 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestResponseCode.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestResponseCode.java
index 19d957aa4..2a76675cc 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestResponseCode.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestResponseCode.java
@@ -30,37 +30,21 @@ public enum StreamingIngestResponseCode {
   ERR_GENERAL_EXCEPTION_RETRY_REQUEST(
       10, "Snowflake experienced a transient exception, please retry the request"),
   ERR_MALFORMED_REQUEST_MISSING_BLOBS_IN_REQUEST(
-      11,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      11, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_MISSING_CHUNKS_FOR_BLOB_IN_REQUEST(
-      12,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      12, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_MISSING_CHANNELS_FOR_CHUNK_FOR_BLOB_IN_REQUEST(
-      13,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      13, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_MISSING_BLOB_PATH(
-      14,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      14, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_MISSING_DATABASE_IN_REQUEST(
-      15,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      15, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_MISSING_SCHEMA_IN_REQUEST(
-      16,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      16, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_MISSING_TABLE_IN_REQUEST(
-      17,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      17, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_MISSING_CHANNEL_NAME_IN_REQUEST(
-      18,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      18, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_CHANNEL_NO_LONGER_EXISTS(
       19,
       "The requested channel no longer exists, most likely due to inactivity. Please re-open the"
@@ -76,60 +60,36 @@ public enum StreamingIngestResponseCode {
       "Another channel managed by this Client had an issue which is preventing the current channel"
           + " from ingesting data. Please re-open the channel"),
   ERR_MALFORMED_REQUEST_DUPLICATE_CHANNEL_IN_CHUNK(
-      23,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      23, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_MISSING_CLIENT_SEQUENCER_IN_REQUEST(
-      24,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      24, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_CHANNEL_DOES_NOT_EXIST_OR_IS_NOT_AUTHORIZED(
       25, "The channel does not exist or is not authorized"),
   ROW_SEQUENCER_IS_COMMITTED(26, "The requested row sequencer is committed"),
   ROW_SEQUENCER_IS_NOT_COMMITTED(27, "The requested row sequencer is not committed"),
 
   ERR_MALFORMED_REQUEST_MISSING_CHUNK_MD5_IN_REQUEST(
-      28,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      28, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_MISSING_EP_INFO_IN_REQUEST(
-      29,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      29, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_INVALID_CHUNK_LENGTH(
-      30,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      30, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_INVALID_ROW_COUNT_IN_EP_INFO(
-      31,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      31, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_INVALID_COLUMN_IN_EP_INFO(
-      32,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      32, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_INVALID_EP_INFO_GENERIC(
-      33,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      33, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_INVALID_BLOB_NAME(
-      34,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      34, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
 
   ERR_CHANNEL_MUST_BE_REOPENED(36, "The channel must be reopened"),
   ERR_MALFORMED_REQUEST_MISSING_ROLE_IN_REQUEST(
-      37,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      37, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_BLOB_HAS_WRONG_FORMAT_OR_EXTENSION(
-      38,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      38, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_BLOB_MISSING_MD5(
-      39,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      39, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MISSING_COLUMN_IN_EP_INFO(
       40,
       "A schema change occurred on the table, please re-open the channel and supply the missing"
@@ -139,13 +99,9 @@ public enum StreamingIngestResponseCode {
   ERR_OUTDATED_CLIENT_SDK_VERSION(
       43, "Outdated Client SDK. Please update your SDK to the latest version"),
   ERR_MALFORMED_REQUEST_UNDEFINED_USER_AGENT_HEADER(
-      44,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      44, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_MALFORMED_REQUEST_MALFORMED_USER_AGENT_HEADER(
-      45,
-      "The SDK generated a malformed request. Please contact Snowflake support for further"
-          + " assistance"),
+      45, StreamingIngestResponseCode.MALFORMED_REQUEST_STATUS_MESSAGE),
   ERR_INTERLEAVING_NOT_SUPPORTED_AT_THIS_TIME(
       46, "Interleaving among tables is not supported at this time"),
   ERR_RESOLVED_TABLE_IS_READ_ONLY(47, "Table is read-only"),
@@ -155,7 +111,12 @@ public enum StreamingIngestResponseCode {
       49,
       "Ingestion into this table is not allowed at this time, please contact Snowflake support");
 
-  public static final String UNKNOWN_STATUS_MESSAGE = "unknown";
+  public static final String UNKNOWN_STATUS_MESSAGE =
+      "Unknown status message. Please contact Snowflake support for further assistance";
+
+  public static final String MALFORMED_REQUEST_STATUS_MESSAGE =
+      "The SDK generated a malformed request. Please contact Snowflake support for further "
+          + " assistance";
 
   private final long statusCode;
 

From 1c58f4bf0d769eb8e3f32d9744f63242756fea7a Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Wed, 14 Jun 2023 23:18:47 +0200
Subject: [PATCH 252/356] NO-SNOW Specify dependencies in public_pom.xml (#527)

---
 public_pom.xml | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/public_pom.xml b/public_pom.xml
index 2edbfb272..9ef308a08 100644
--- a/public_pom.xml
+++ b/public_pom.xml
@@ -34,4 +34,19 @@
       <connection>scm:git:git://github.com/snowflakedb/snowflake-ingest-java</connection>
       <url>http://github.com/snowflakedb/snowflake-ingest-java/tree/master</url>
     </scm>
+
+    <dependencies>
+        <dependency>
+            <groupId>net.snowflake</groupId>
+            <artifactId>snowflake-jdbc</artifactId>
+            <version>3.13.30</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.36</version>
+            <scope>compile</scope>
+        </dependency>
+    </dependencies>
 </project>

From c284767c2653ebb180a41ea64848189c9537e6e7 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 14 Jun 2023 15:23:16 -0700
Subject: [PATCH 253/356] V2.0.1 release (#528)

This is a patch release on top of the v2.0.0 where we fix an unexpected dependency behavior for Snowflake JDBC
---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 021808e0b..2d810f0e4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.0.0</version>
+  <version>2.0.1</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index a053047bf..641e5b673 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -119,7 +119,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.0.0";
+  public static final String DEFAULT_VERSION = "2.0.1";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From f9071f94915fc270e51764ba0b39b8cf034198d5 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Fri, 16 Jun 2023 11:17:05 -0700
Subject: [PATCH 254/356] Remove CLA bot and update README.md to include cla
 and formatter requirements (#510)

* try whitesource token

* Remove cla workflow

* Update README.md with cla and formatting requirements

* Update README.md

---------

Co-authored-by: Xin Huang <xin.huang@snowflake.com>
---
 .github/workflows/cla_bot.yml | 24 ------------------------
 README.md                     | 12 +++++++-----
 2 files changed, 7 insertions(+), 29 deletions(-)
 delete mode 100644 .github/workflows/cla_bot.yml

diff --git a/.github/workflows/cla_bot.yml b/.github/workflows/cla_bot.yml
deleted file mode 100644
index bfd7609f0..000000000
--- a/.github/workflows/cla_bot.yml
+++ /dev/null
@@ -1,24 +0,0 @@
-name: "CLA Assistant"
-on:
-  issue_comment:
-    types: [created]
-  pull_request_target:
-    types: [opened,closed,synchronize]
-
-jobs:
-  CLAssistant:
-    runs-on: ubuntu-latest
-    steps:
-      - name: "CLA Assistant"
-        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
-        uses: contributor-assistant/github-action/@master
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_BOT_TOKEN }}
-        with:
-          path-to-signatures: 'signatures/version1.json'
-          path-to-document: 'https://github.com/Snowflake-Labs/CLA/blob/main/README.md'
-          branch: 'main'
-          allowlist: 'dependabot[bot],github-actions,Jenkins User,sfc-gh-snyk-sca-sa'
-          remote-organization-name: 'snowflakedb'
-          remote-repository-name: 'cla-db'
diff --git a/README.md b/README.md
index 97e1bca92..11704baa6 100644
--- a/README.md
+++ b/README.md
@@ -115,9 +115,11 @@ console log output.
     -   Here is the link for documentation [Key Pair
             Generator](https://docs.snowflake.com/en/user-guide/key-pair-auth.html)
 
-# Code style
+# Contributing to this repo
 
-We use [Google Java format](https://github.com/google/google-java-format) to format the code. To format all files, run:
-```bash
-./format.sh
-```
+Each PR must pass all required github action merge gates before approval and merge. In addition to those tests, you will need:
+
+- Formatter: run this script [`./format.sh`](https://github.com/snowflakedb/snowflake-ingest-java/blob/master/format.sh) from root
+- CLA: all contributers must sign the Snowflake CLA. This is a one time signature, please provide your email so we can work with you to get this signed after you open a PR.
+
+Thank you for contributing! We will review and approve PRs as soon as we can.

From 102410b38225c3fbb1b8fb6d1b69437c662c8879 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Tue, 20 Jun 2023 10:39:37 +0200
Subject: [PATCH 255/356] SNOW-841883 Upgrade snappy-java to 1.1.10.1 (#533)

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 2d810f0e4..d1ecb43a1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -63,7 +63,7 @@
     <protobuf.version>3.19.6</protobuf.version>
     <shadeBase>net.snowflake.ingest.internal</shadeBase>
     <slf4j.version>1.7.36</slf4j.version>
-    <snappy.version>1.1.8.3</snappy.version>
+    <snappy.version>1.1.10.1</snappy.version>
     <snowjdbc.version>3.13.30</snowjdbc.version>
     <yetus.version>0.13.0</yetus.version>
   </properties>

From 0ae93a4114985373b1af6914d8923616d88e5451 Mon Sep 17 00:00:00 2001
From: sfc-gh-snyk-sca-sa
 <95290361+sfc-gh-snyk-sca-sa@users.noreply.github.com>
Date: Tue, 20 Jun 2023 16:27:25 +0530
Subject: [PATCH 256/356] [Snyk] Security upgrade com.google.guava:guava from
 31.1-jre to 32.0.0-jre (#534)

fix: pom.xml to reduce vulnerabilities

The following vulnerabilities are fixed with an upgrade:
- https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEGUAVA-5710356

Co-authored-by: snyk-bot <snyk-bot@snyk.io>
Co-authored-by: Lukas Sembera <lukas.sembera@snowflake.com>
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index d1ecb43a1..d97897949 100644
--- a/pom.xml
+++ b/pom.xml
@@ -44,7 +44,7 @@
     <commonslogging.version>1.2</commonslogging.version>
     <commonstext.version>1.10.0</commonstext.version>
     <fasterxml.version>2.14.0</fasterxml.version>
-    <guava.version>31.1-jre</guava.version>
+    <guava.version>32.0.0-jre</guava.version>
     <hadoop.version>3.3.5</hadoop.version>
     <jacoco.skip.instrument>true</jacoco.skip.instrument>
     <jacoco.version>0.8.5</jacoco.version>

From afed6bb6a36edd7be28a3873ece5a4330602e76f Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Tue, 20 Jun 2023 21:49:55 +0200
Subject: [PATCH 257/356] NO-SNOW Bump version to 2.0.2-SNAPSHOT (#529)

---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index d97897949..98a78f626 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.0.1</version>
+  <version>2.0.2-SNAPSHOT</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 641e5b673..5893291a3 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -119,7 +119,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.0.1";
+  public static final String DEFAULT_VERSION = "2.0.2-SNAPSHOT";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 076cd5a71a5a4d5de1da2215690d47f063d09826 Mon Sep 17 00:00:00 2001
From: Alec Huang <alec.huang@snowflake.com>
Date: Tue, 20 Jun 2023 16:17:44 -0700
Subject: [PATCH 258/356] SNOW-352846 OAuth Authentication: #1 Polymorph auth
 credential (#532)

Preliminary change of upcoming OAuth authentication support. This does not change any behavior.
---
 .../ingest/connection/JWTManager.java         | 180 ++++++++++++++++
 .../ingest/connection/RequestBuilder.java     |   2 +-
 .../ingest/connection/SecurityManager.java    | 202 +++---------------
 .../ingest/connection/TelemetryService.java   |   2 +-
 .../connection/SecurityManagerTest.java       |  86 ++++++++
 .../ingest/connection/TestKeyRenewal.java     |  49 -----
 6 files changed, 293 insertions(+), 228 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/connection/JWTManager.java
 create mode 100644 src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java
 delete mode 100644 src/test/java/net/snowflake/ingest/connection/TestKeyRenewal.java

diff --git a/src/main/java/net/snowflake/ingest/connection/JWTManager.java b/src/main/java/net/snowflake/ingest/connection/JWTManager.java
new file mode 100644
index 000000000..e5ee3aa0b
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/connection/JWTManager.java
@@ -0,0 +1,180 @@
+/*
+ * Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.connection;
+
+import com.nimbusds.jose.JOSEException;
+import com.nimbusds.jose.JWSAlgorithm;
+import com.nimbusds.jose.JWSHeader;
+import com.nimbusds.jose.JWSSigner;
+import com.nimbusds.jose.crypto.RSASSASigner;
+import com.nimbusds.jwt.JWTClaimsSet;
+import com.nimbusds.jwt.SignedJWT;
+import java.security.KeyPair;
+import java.util.Date;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import net.snowflake.ingest.utils.Cryptor;
+
+/**
+ * This class manages creating and automatically renewing the JWT token
+ *
+ * @author obabarinsa
+ */
+final class JWTManager extends SecurityManager {
+  // the token lifetime is 59 minutes
+  private static final float LIFETIME_IN_MINUTES = 59;
+
+  // the renewal time is 54 minutes
+  private static final int RENEWAL_INTERVAL_IN_MINUTES = 54;
+
+  private static final String TOKEN_TYPE = "KEYPAIR_JWT";
+
+  // The public - private key pair we're using to connect to the service
+  private final transient KeyPair keyPair;
+
+  // the token itself
+  protected final AtomicReference<String> token;
+
+  /**
+   * Creates a JWTManager entity for a given account, user and KeyPair with a specified time to
+   * renew the token
+   *
+   * @param accountName - the snowflake account name of this user
+   * @param username - the snowflake username of the current user
+   * @param keyPair - the public/private key pair we're using to connect
+   * @param timeTillRenewal - the time measure until we renew the token
+   * @param unit the unit by which timeTillRenewal is measured
+   * @param telemetryService reference to the telemetry service
+   */
+  JWTManager(
+      String accountName,
+      String username,
+      KeyPair keyPair,
+      int timeTillRenewal,
+      TimeUnit unit,
+      TelemetryService telemetryService) {
+    super(accountName, username, telemetryService);
+    // if any of our arguments are null, throw an exception
+    if (keyPair == null) {
+      throw new IllegalArgumentException();
+    }
+    token = new AtomicReference<>();
+
+    // we have to keep around the keys
+    this.keyPair = keyPair;
+
+    // generate our first token
+    regenerateToken();
+
+    // schedule all future renewals
+    tokenRefresher.scheduleAtFixedRate(
+        this::regenerateToken, timeTillRenewal, timeTillRenewal, unit);
+  }
+
+  /**
+   * Creates a JWTManager entity for a given account, user and KeyPair with the default time to
+   * renew (RENEWAL_INTERVAL_IN_MINUTES minutes)
+   *
+   * @param accountName - the snowflake account name of this user
+   * @param username - the snowflake username of the current user
+   * @param keyPair - the public/private key pair we're using to connect
+   * @param telemetryService reference to the telemetry service
+   */
+  JWTManager(
+      String accountName, String username, KeyPair keyPair, TelemetryService telemetryService) {
+    this(
+        accountName,
+        username,
+        keyPair,
+        RENEWAL_INTERVAL_IN_MINUTES,
+        TimeUnit.MINUTES,
+        telemetryService);
+  }
+
+  @Override
+  String getToken() {
+    // if we failed to regenerate the token at some point, throw
+    if (refreshFailed.get()) {
+      LOGGER.error("getToken request failed due to token regeneration failure");
+      throw new SecurityException();
+    }
+
+    return token.get();
+  }
+
+  @Override
+  String getTokenType() {
+    return TOKEN_TYPE;
+  }
+
+  /** regenerateToken - Regenerates our Token given our current user, account and keypair */
+  private void regenerateToken() {
+    // create our JWT claim builder object
+    JWTClaimsSet.Builder builder = new JWTClaimsSet.Builder();
+
+    // get the subject to the fully qualified username
+    String subject = String.format("%s.%s", account, user);
+
+    // get the issuer
+    String publicKeyFPInJwt = calculatePublicKeyFp(keyPair);
+    String issuer = String.format("%s.%s.%s", account, user, publicKeyFPInJwt);
+
+    // iat set to now
+    Date iat = new Date(System.currentTimeMillis());
+
+    // expiration in 59 minutes
+    Date exp = new Date(iat.getTime() + 59 * 60 * 1000);
+
+    // build claim set
+    JWTClaimsSet claimsSet =
+        builder.issuer(issuer).subject(subject).issueTime(iat).expirationTime(exp).build();
+    LOGGER.debug("Creating new JWT with subject {} and issuer {}...", subject, issuer);
+
+    SignedJWT signedJWT = new SignedJWT(new JWSHeader(JWSAlgorithm.RS256), claimsSet);
+
+    JWSSigner signer = new RSASSASigner(this.keyPair.getPrivate());
+
+    String newToken;
+    try {
+      signedJWT.sign(signer);
+      newToken = signedJWT.serialize();
+    } catch (JOSEException e) {
+      refreshFailed.set(true);
+      LOGGER.error("Failed to regenerate token! Exception is as follows : {}", e.getMessage());
+      throw new SecurityException();
+    }
+
+    // atomically update the string
+    LOGGER.info("Successfully created new JWT");
+    token.set(newToken);
+
+    // Refresh the token used in the telemetry service as well
+    if (telemetryService != null) {
+      telemetryService.refreshToken(newToken);
+    }
+  }
+
+  /**
+   * Given a keypair
+   *
+   * @return the fingerprint of public key
+   *     <p>The idea is to hash public key's raw bytes using SHA-256 and converts hash into a string
+   *     using Base64 encoding.
+   */
+  private String calculatePublicKeyFp(KeyPair keyPair) {
+    // get the raw bytes of public key
+    byte[] publicKeyRawBytes = keyPair.getPublic().getEncoded();
+
+    // take sha256 on raw bytes and do base64 encode
+    publicKeyFingerPrint = String.format("SHA256:%s", Cryptor.sha256HashBase64(publicKeyRawBytes));
+    return publicKeyFingerPrint;
+  }
+
+  /** Currently, it only shuts down the instance of ExecutorService. */
+  @Override
+  public void close() {
+    tokenRefresher.shutdown();
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 5893291a3..35bca3852 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -267,7 +267,7 @@ public RequestBuilder(
             : null;
 
     // create our security/token manager
-    securityManager = new SecurityManager(accountName, userName, keyPair, telemetryService);
+    securityManager = new JWTManager(accountName, userName, keyPair, telemetryService);
 
     // stash references to the account and user name as well
     String account = accountName.toUpperCase();
diff --git a/src/main/java/net/snowflake/ingest/connection/SecurityManager.java b/src/main/java/net/snowflake/ingest/connection/SecurityManager.java
index 96bc15e59..fd6b3daff 100644
--- a/src/main/java/net/snowflake/ingest/connection/SecurityManager.java
+++ b/src/main/java/net/snowflake/ingest/connection/SecurityManager.java
@@ -1,141 +1,71 @@
 /*
- * Copyright (c) 2012-2017 Snowflake Computing Inc. All rights reserved.
+ * Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved.
  */
 
 package net.snowflake.ingest.connection;
 
-import com.nimbusds.jose.JOSEException;
-import com.nimbusds.jose.JWSAlgorithm;
-import com.nimbusds.jose.JWSHeader;
-import com.nimbusds.jose.JWSSigner;
-import com.nimbusds.jose.crypto.RSASSASigner;
-import com.nimbusds.jwt.JWTClaimsSet;
-import com.nimbusds.jwt.SignedJWT;
-import java.security.KeyPair;
-import java.util.Date;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-import net.snowflake.ingest.utils.Cryptor;
 import net.snowflake.ingest.utils.ThreadFactoryUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * This class manages creating and automatically renewing the JWT token
+ * This class manages creating and automatically renewing the token
  *
  * @author obabarinsa
- * @since 1.8
  */
-final class SecurityManager implements AutoCloseable {
+abstract class SecurityManager implements AutoCloseable {
   // the logger for SecurityManager
-  private static final Logger LOGGER = LoggerFactory.getLogger(SecurityManager.class);
+  protected static final Logger LOGGER = LoggerFactory.getLogger(SecurityManager.class);
 
-  // the token lifetime is 59 minutes
-  private static final float LIFETIME = 59;
-
-  // the renewal time is 54 minutes
-  private static final int RENEWAL_INTERVAL = 54;
-
-  // The public - private key pair we're using to connect to the service
-  private final transient KeyPair keyPair;
-
-  // the name of the account on behalf of which we're connecting
-  private final String account;
+  protected final String account;
 
   // Fingerprint of public key sent from client in jwt payload
-  private String publicKeyFingerPrint;
+  protected String publicKeyFingerPrint;
 
   // the name of the user who will be loading the files
-  private final String user;
+  protected final String user;
 
-  // the token itself
-  private final AtomicReference<String> token;
-
-  // Did we fail to regenerate our token at some point?
-  private final AtomicBoolean regenFailed;
+  // Did we fail to refresh our token at some point?
+  protected final AtomicBoolean refreshFailed;
 
   // Thread factory for daemon threads so that application can shutdown
   final ThreadFactory tf = ThreadFactoryUtil.poolThreadFactory(getClass().getSimpleName(), true);
 
-  // the thread we use for renewing all tokens
-  private final ScheduledExecutorService keyRenewer = Executors.newScheduledThreadPool(1, tf);
+  // the thread we use for refresh tokens
+  protected ScheduledExecutorService tokenRefresher = Executors.newScheduledThreadPool(1, tf);
 
   // Reference to the Telemetry Service in the client
-  private final TelemetryService telemetryService;
+  protected final TelemetryService telemetryService;
 
   /**
-   * Creates a SecurityManager entity for a given account, user and KeyPair with a specified time to
-   * renew the token
+   * Creates a SecurityManager entity for a given account
    *
    * @param accountName - the snowflake account name of this user
    * @param username - the snowflake username of the current user
-   * @param keyPair - the public/private key pair we're using to connect
-   * @param timeTillRenewal - the time measure until we renew the token
-   * @param unit the unit by which timeTillRenewal is measured
-   * @param telemetryService reference to the telemetry service
    */
-  SecurityManager(
-      String accountName,
-      String username,
-      KeyPair keyPair,
-      int timeTillRenewal,
-      TimeUnit unit,
-      TelemetryService telemetryService) {
+  SecurityManager(String accountName, String username, TelemetryService telemetryService) {
     // if any of our arguments are null, throw an exception
-    if (accountName == null || username == null || keyPair == null) {
+    if (accountName == null || username == null) {
       throw new IllegalArgumentException();
     }
     account = parseAccount(accountName);
     user = username.toUpperCase();
 
-    // create our automatic reference to a string (our token)
-    token = new AtomicReference<>();
-
     // we haven't yet failed to regenerate our token
-    regenFailed = new AtomicBoolean();
-
-    // we have to keep around the keys
-    this.keyPair = keyPair;
+    this.refreshFailed = new AtomicBoolean();
 
     this.telemetryService = telemetryService;
-
-    // generate our first token
-    regenerateToken();
-
-    // schedule all future renewals
-    keyRenewer.scheduleAtFixedRate(this::regenerateToken, timeTillRenewal, timeTillRenewal, unit);
   }
 
-  /**
-   * Creates a SecurityManager entity for a given account, user and KeyPair with the default time to
-   * renew (RENEWAL_INTERVAL minutes)
-   *
-   * @param accountName - the snowflake account name of this user
-   * @param username - the snowflake username of the current user
-   * @param keyPair - the public/private key pair we're using to connect
-   * @param telemetryService reference to the telemetry service
-   */
-  SecurityManager(
-      String accountName, String username, KeyPair keyPair, TelemetryService telemetryService) {
-    this(accountName, username, keyPair, RENEWAL_INTERVAL, TimeUnit.MINUTES, telemetryService);
+  /** Test only */
+  void setRefreshFailed(Boolean refreshFailed) {
+    this.refreshFailed.set(refreshFailed);
   }
 
-  /**
-   * Trims an account name if it contains a <b>"."</b>
-   *
-   * <p>Snowflake's python connector trims an accountName if it contains a "."
-   *
-   * @param accountName given accountName in SimpleIngestManager's constructor
-   * @return initial part of account name if originally it contained a ".", otherwise return the
-   *     same accountName.
-   * @see <a
-   *     href="https://github.com/snowflakedb/snowflake-connector-python/blob/master/src/snowflake/connector/util_text.py#L227">Python
-   *     Connector</a>
-   */
   private String parseAccount(final String accountName) {
     String parseAccount = null;
     if (accountName.contains(".")) {
@@ -149,98 +79,16 @@ private String parseAccount(final String accountName) {
     return parseAccount.toUpperCase();
   }
 
-  /** regenerateToken - Regenerates our Token given our current user, account and keypair */
-  private void regenerateToken() {
-    // create our JWT claim builder object
-    JWTClaimsSet.Builder builder = new JWTClaimsSet.Builder();
-
-    // get the subject to the fully qualified username
-    String subject = String.format("%s.%s", account, user);
-
-    // get the issuer
-    String publicKeyFPInJwt = calculatePublicKeyFp(keyPair);
-    String issuer = String.format("%s.%s.%s", account, user, publicKeyFPInJwt);
-
-    // iat set to now
-    Date iat = new Date(System.currentTimeMillis());
-
-    // expiration in 59 minutes
-    Date exp = new Date(iat.getTime() + 59 * 60 * 1000);
-
-    // build claim set
-    JWTClaimsSet claimsSet =
-        builder.issuer(issuer).subject(subject).issueTime(iat).expirationTime(exp).build();
-    LOGGER.debug("Creating new JWT with subject {} and issuer {}...", subject, issuer);
-
-    SignedJWT signedJWT = new SignedJWT(new JWSHeader(JWSAlgorithm.RS256), claimsSet);
-
-    JWSSigner signer = new RSASSASigner(this.keyPair.getPrivate());
-
-    String newToken;
-    try {
-      signedJWT.sign(signer);
-      newToken = signedJWT.serialize();
-    } catch (JOSEException e) {
-      regenFailed.set(true);
-      LOGGER.error("Failed to regenerate token! Exception is as follows : {}", e.getMessage());
-      throw new SecurityException();
-    }
-
-    // atomically update the string
-    LOGGER.info("Successfully created new JWT");
-    token.set(newToken);
-
-    // Refresh the token used in the telemetry service as well
-    if (telemetryService != null) {
-      telemetryService.refreshJWTToken(newToken);
-    }
-  }
-
   /**
    * getToken - returns we've most recently generated
    *
-   * @return the string version of a valid JWT token
-   * @throws SecurityException if we failed to regenerate a token since the last call
-   */
-  String getToken() {
-    // if we failed to regenerate the token at some point, throw
-    if (regenFailed.get()) {
-      LOGGER.error("getToken request failed due to token regeneration failure");
-      throw new SecurityException();
-    }
-
-    return token.get();
-  }
-
-  /* Only used in testing at the moment */
-  String getAccount() {
-    return this.account;
-  }
-
-  /**
-   * Given a keypair
-   *
-   * @return the fingerprint of public key
-   *     <p>The idea is to hash public key's raw bytes using SHA-256 and converts hash into a string
-   *     using Base64 encoding.
+   * @return the string version of a valid token
+   * @throws SecurityException if we failed to regenerate or refresh a token since the last call
    */
-  private String calculatePublicKeyFp(KeyPair keyPair) {
-    // get the raw bytes of public key
-    byte[] publicKeyRawBytes = keyPair.getPublic().getEncoded();
+  abstract String getToken();
 
-    // take sha256 on raw bytes and do base64 encode
-    publicKeyFingerPrint = String.format("SHA256:%s", Cryptor.sha256HashBase64(publicKeyRawBytes));
-    return publicKeyFingerPrint;
-  }
+  /** getTokenType - returns the token type, either "KEYPAIR_JWT" or "OAUTH" */
+  abstract String getTokenType();
 
-  /** Only called by SecurityManagerTest */
-  String getPublicKeyFingerPrint() {
-    return publicKeyFingerPrint;
-  }
-
-  /** Currently, it only shuts down the instance of ExecutorService. */
-  @Override
-  public void close() {
-    keyRenewer.shutdown();
-  }
+  public abstract void close();
 }
diff --git a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java
index a330ff3c8..ee1e1353d 100644
--- a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java
+++ b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java
@@ -148,7 +148,7 @@ private ObjectNode buildMsgFromTimer(Timer timer) {
   }
 
   /** Refresh JWT token stored in the telemetry client */
-  public void refreshJWTToken(String token) {
+  public void refreshToken(String token) {
     telemetry.refreshToken(token);
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java b/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java
new file mode 100644
index 000000000..94d83f8eb
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java
@@ -0,0 +1,86 @@
+package net.snowflake.ingest.connection;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.NoSuchAlgorithmException;
+import java.util.concurrent.TimeUnit;
+import org.junit.Before;
+import org.junit.Test;
+
+/** SecurityManager - tests functionally of security manager */
+public class SecurityManagerTest {
+  // generate our keys using RSA
+  private static final String ALGORITHM = "RSA";
+  private KeyPair keyPair;
+
+  @Before
+  public void setup() throws NoSuchAlgorithmException {
+    // first we need to create a keypair
+    KeyPairGenerator keyGen = KeyPairGenerator.getInstance(ALGORITHM);
+    // we need a 2048 bit RSA key for this test
+    keyGen.initialize(2048);
+    // generate the actual keys
+    keyPair = keyGen.generateKeyPair();
+  }
+
+  /** Test instantiate security manager with invalid params */
+  @Test
+  public void testSecurityManagerInstantiate() {
+    // Try to create jwt manager without required args
+    try {
+      SecurityManager jwtManager = new JWTManager(null, null, null, 3, TimeUnit.SECONDS, null);
+    } catch (IllegalArgumentException e1) {
+      try {
+        SecurityManager jwtManager =
+            new JWTManager("account", "user", null, 3, TimeUnit.SECONDS, null);
+      } catch (IllegalArgumentException e2) {
+        return;
+      }
+    }
+    assertFalse("testSecurityManagerInstantiate failed", true);
+  }
+
+  /** Test get token type, should exactly match the request */
+  @Test
+  public void testGetTokenType() {
+    SecurityManager manager = new JWTManager("account", "user", keyPair, 3, TimeUnit.SECONDS, null);
+    assertEquals(manager.getTokenType(), "KEYPAIR_JWT");
+  }
+
+  /** Evaluates whether or not we are actually renewing jwt tokens */
+  @Test
+  public void doesRegenerateJWTToken() throws InterruptedException {
+    // create the security manager;
+    SecurityManager manager = new JWTManager("account", "user", keyPair, 3, TimeUnit.SECONDS, null);
+
+    // grab a token
+    String token = manager.getToken();
+
+    // if we immediately request the same token, it should be exactly the same
+    assertEquals(
+        "Two token requests prior to renewals should be the same", token, manager.getToken());
+
+    // sleep for enough time for the thread renewal thread to run
+    Thread.sleep(6000);
+
+    // ascertain that we have overwritten the token
+    assertNotEquals("The renewal thread should have reset the token", token, manager.getToken());
+  }
+
+  /** Test behavior of getting token after refresh failed */
+  @Test
+  public void testGetTokenFail() {
+    SecurityManager manager = new JWTManager("account", "user", keyPair, 3, TimeUnit.SECONDS, null);
+    manager.setRefreshFailed(true);
+    try {
+      String token = manager.getToken();
+    } catch (SecurityException e) {
+      return;
+    }
+    assertFalse("testGetTokenFail failed", true);
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/connection/TestKeyRenewal.java b/src/test/java/net/snowflake/ingest/connection/TestKeyRenewal.java
deleted file mode 100644
index 9bc653c39..000000000
--- a/src/test/java/net/snowflake/ingest/connection/TestKeyRenewal.java
+++ /dev/null
@@ -1,49 +0,0 @@
-package net.snowflake.ingest.connection;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-
-import java.security.KeyPair;
-import java.security.KeyPairGenerator;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.util.concurrent.TimeUnit;
-import org.junit.Test;
-
-/**
- * TestKeyRenewal - tests whether or not the SecurityManager is successfully resetting the the JWT
- * Token after the specified timespan
- */
-public class TestKeyRenewal {
-  // generate our keys using RSA
-  private static final String ALGORITHM = "RSA";
-
-  /** Evaluates whether or not we are actually renewing tokens */
-  @Test
-  public void doesRegenerateToken()
-      throws NoSuchProviderException, NoSuchAlgorithmException, InterruptedException {
-    // first we need to create a keypair
-    KeyPairGenerator keyGen = KeyPairGenerator.getInstance(ALGORITHM);
-    // we need a 2048 bit RSA key for this test
-    keyGen.initialize(2048);
-    // generate the actual keys
-    KeyPair keyPair = keyGen.generateKeyPair();
-
-    // create the security manager;
-    SecurityManager manager =
-        new SecurityManager("account", "user", keyPair, 3, TimeUnit.SECONDS, null);
-
-    // grab a token
-    String token = manager.getToken();
-
-    // if we immediately request the same token, it should be exactly the same
-    assertEquals(
-        "Two token requests prior to renewals should be the same", token, manager.getToken());
-
-    // sleep for enough time for the thread renewal thread to run
-    Thread.sleep(6000);
-
-    // ascertain that we have overwritten the token
-    assertNotEquals("The renewal thread should have reset the token", token, manager.getToken());
-  }
-}

From 82a3ecf162a078b2fd26052b96fb485cbfb54677 Mon Sep 17 00:00:00 2001
From: Jay Patel <jay.patel@snowflake.com>
Date: Wed, 21 Jun 2023 20:22:46 +0000
Subject: [PATCH 259/356] SNOW-693813 disable client info snowpipe tests,
 remove later (#536)

NO-SNOW disable client info snowpipe tests, remove later
---
 src/test/java/net/snowflake/ingest/SimpleIngestIT.java | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
index 0617898c8..b8740ee2e 100644
--- a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
@@ -33,6 +33,7 @@
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
 /** Example ingest sdk integration test */
@@ -384,6 +385,7 @@ private void verifyDefaultUserAgent(
   }
 
   @Test
+  @Ignore("SNOW-693813") // Disable Snowpipe Exactly once, remove later
   public void testConfigureClientHappyCase() throws Exception {
     final String userAgentSuffix = "kafka-provider/NONE";
     SimpleIngestManager manager = TestUtils.getManager(pipeName, userAgentSuffix);
@@ -392,6 +394,7 @@ public void testConfigureClientHappyCase() throws Exception {
   }
 
   @Test
+  @Ignore("SNOW-693813") // Disable Snowpipe Exactly once, remove later
   public void testConfigureClientNoPipeFound() throws Exception {
     final String userAgentSuffix = "kafka-provider/NONE";
     SimpleIngestManager manager = TestUtils.getManager("nopipe", userAgentSuffix);
@@ -406,6 +409,7 @@ public void testConfigureClientNoPipeFound() throws Exception {
   }
 
   @Test
+  @Ignore("SNOW-693813") // Disable Snowpipe Exactly once, remove later
   public void testGetClientStatusHappyCase() throws Exception {
     final String userAgentSuffix = "kafka-provider/NONE";
     SimpleIngestManager manager = TestUtils.getManager(pipeName, userAgentSuffix);
@@ -416,6 +420,7 @@ public void testGetClientStatusHappyCase() throws Exception {
   }
 
   @Test
+  @Ignore("SNOW-693813") // Disable Snowpipe Exactly once, remove later
   public void testGetClientStatusNoPipeFound() throws Exception {
     final String userAgentSuffix = "kafka-provider/NONE";
     SimpleIngestManager manager = TestUtils.getManager("nopipe", userAgentSuffix);
@@ -430,6 +435,7 @@ public void testGetClientStatusNoPipeFound() throws Exception {
   }
 
   @Test
+  @Ignore("SNOW-693813") // Disable Snowpipe Exactly once, remove later
   public void testIngestFilesWithClientInfo() throws Exception {
 
     // first lets call configure client API
@@ -465,6 +471,7 @@ public void testIngestFilesWithClientInfo() throws Exception {
   }
 
   @Test
+  @Ignore("SNOW-693813") // Disable Snowpipe Exactly once, remove later
   public void testIngestFilesWithClientInfoWithOldClientSequencer() throws Exception {
 
     // first lets call configure client API
@@ -526,6 +533,7 @@ public void testIngestFilesWithClientInfoWithOldClientSequencer() throws Excepti
   }
 
   @Test
+  @Ignore("SNOW-693813") // Disable Snowpipe Exactly once, remove later
   public void testIngestFilesWithClientInfoWithNoClientSequencer() throws Exception {
     // first lets call configure client API
     final String userAgentSuffix = "kafka-provider/NONE";

From 96724741df0050785241747cf2debdd936c66bb0 Mon Sep 17 00:00:00 2001
From: Jay Patel <jay.patel@snowflake.com>
Date: Wed, 21 Jun 2023 21:43:23 +0000
Subject: [PATCH 260/356] Use later version of google guava, as recommended by
 CVE (#535)

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 98a78f626..471f4fd37 100644
--- a/pom.xml
+++ b/pom.xml
@@ -44,7 +44,7 @@
     <commonslogging.version>1.2</commonslogging.version>
     <commonstext.version>1.10.0</commonstext.version>
     <fasterxml.version>2.14.0</fasterxml.version>
-    <guava.version>32.0.0-jre</guava.version>
+    <guava.version>32.0.1-jre</guava.version>
     <hadoop.version>3.3.5</hadoop.version>
     <jacoco.skip.instrument>true</jacoco.skip.instrument>
     <jacoco.version>0.8.5</jacoco.version>

From c9cc3aa1e238f61e5c92a257c6eef93ecf2edc4c Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Fri, 23 Jun 2023 08:50:14 -0700
Subject: [PATCH 261/356] [SNOW-843760] Upgrade netty-common in response to CVE
 vulnerability  (#538)

upgrade netty
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 471f4fd37..c3994eb00 100644
--- a/pom.xml
+++ b/pom.xml
@@ -55,7 +55,7 @@
     <maven.compiler.source>1.8</maven.compiler.source>
     <maven.compiler.target>1.8</maven.compiler.target>
     <net.minidev.version>2.4.9</net.minidev.version>
-    <netty.version>4.1.82.Final</netty.version>
+    <netty.version>4.1.94.Final</netty.version>
     <nimbusds.version>9.9.3</nimbusds.version>
     <objenesis.version>3.1</objenesis.version>
     <parquet.version>1.12.3</parquet.version>

From 2f98903d6008a9e04795544285b9f91c04a19885 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Mon, 26 Jun 2023 15:20:00 +0200
Subject: [PATCH 262/356] NO-SNOW Add note about dependencies (#540)

---
 README.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/README.md b/README.md
index 11704baa6..ba4a1f813 100644
--- a/README.md
+++ b/README.md
@@ -11,6 +11,16 @@ authentication. Currently, we support ingestion through the following APIs:
 1. [Snowpipe](https://docs.snowflake.com/en/user-guide/data-load-snowpipe-rest-gs.html#client-requirement-java-or-python-sdk)
 2. [Snowpipe Streaming](https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview) - Under Public Preview
 
+# Dependencies
+
+The Snowflake Ingest Service SDK depends on the following libraries:
+
+* snowflake-jdbc (3.13.30+)
+* slf4j-api
+
+These dependencies will be fetched automatically by build systems like Maven or Gradle. If you don't build your project
+using a build system, please make sure these dependencies are on the classpath.
+
 # Prerequisites
 
 **If your project depends on the Snowflake JDBC driver, as well, please make sure the JDBC driver version is 3.13.30 or newer.**

From 526cffad8ffe226e60eee8b354df61a1d36eb0ee Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Wed, 28 Jun 2023 12:50:06 -0700
Subject: [PATCH 263/356] [SNOW-693813] Remove exactly once methods calling
 into server (#539)

* remove corresponding kc methods

* autoformatting

* remove handler calls

* autoformatting
---
 .../snowflake/ingest/SimpleIngestManager.java |  88 +--------
 .../connection/ClientStatusResponse.java      |  44 -----
 .../connection/ConfigureClientResponse.java   |  26 ---
 .../connection/InsertFilesClientInfo.java     |  75 -------
 .../ingest/connection/RequestBuilder.java     | 160 +--------------
 .../connection/ServiceResponseHandler.java    |  60 ------
 .../net/snowflake/ingest/SimpleIngestIT.java  | 185 ------------------
 7 files changed, 8 insertions(+), 630 deletions(-)
 delete mode 100644 src/main/java/net/snowflake/ingest/connection/ClientStatusResponse.java
 delete mode 100644 src/main/java/net/snowflake/ingest/connection/ConfigureClientResponse.java
 delete mode 100644 src/main/java/net/snowflake/ingest/connection/InsertFilesClientInfo.java

diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
index d167488dd..aa8a00bf9 100644
--- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
+++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
@@ -15,7 +15,6 @@
 import java.security.spec.InvalidKeySpecException;
 import java.util.Collections;
 import java.util.List;
-import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
@@ -23,13 +22,10 @@
 import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpGet;
 import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost;
 import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
-import net.snowflake.ingest.connection.ClientStatusResponse;
-import net.snowflake.ingest.connection.ConfigureClientResponse;
 import net.snowflake.ingest.connection.HistoryRangeResponse;
 import net.snowflake.ingest.connection.HistoryResponse;
 import net.snowflake.ingest.connection.IngestResponse;
 import net.snowflake.ingest.connection.IngestResponseException;
-import net.snowflake.ingest.connection.InsertFilesClientInfo;
 import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.connection.ServiceResponseHandler;
 import net.snowflake.ingest.utils.BackOffException;
@@ -524,51 +520,17 @@ public IngestResponse ingestFiles(List<StagedFileWrapper> files, UUID requestId)
   public IngestResponse ingestFiles(
       List<StagedFileWrapper> files, UUID requestId, boolean showSkippedFiles)
       throws URISyntaxException, IOException, IngestResponseException, BackOffException {
-    return ingestFiles(files, requestId, showSkippedFiles, null /* Client info is null */);
-  }
-
-  /**
-   * ingestFiles With Client Info - synchronously sends a request to the ingest service to enqueue
-   * these files along with clientSequencer and offSetToken.
-   *
-   * <p>OffsetToken will be atomically persisted on server(Snowflake) side along with files if the
-   * clientSequencer added in this request matches with what Snowflake currently has.
-   *
-   * <p>If clientSequencers doesnt match, 400 response code is sent back and no files will be added.
-   *
-   * @param files - list of wrappers around filenames and sizes
-   * @param requestId - a requestId that we'll use to label - if null, we generate one for the user
-   * @param showSkippedFiles - a flag which returns the files that were skipped when set to true.
-   * @param clientInfo - clientSequencer and offsetToken to pass along with files. Can be null.
-   * @return an insert response from the server
-   * @throws URISyntaxException - if the provided account name was illegal and caused a URI
-   *     construction failure
-   * @throws IOException - if we have some other network failure
-   * @throws IngestResponseException - if snowflake encountered error during ingest
-   * @throws BackOffException - if we have a 503 response
-   */
-  public IngestResponse ingestFiles(
-      List<StagedFileWrapper> files,
-      UUID requestId,
-      boolean showSkippedFiles,
-      InsertFilesClientInfo clientInfo)
-      throws URISyntaxException, IOException, IngestResponseException, BackOffException {
-
     // the request id we want to send with this payload
     if (requestId == null || requestId.toString().isEmpty()) {
       requestId = UUID.randomUUID();
     }
 
     HttpPost httpPostForIngestFile =
-        builder.generateInsertRequest(
-            requestId, pipe, files, showSkippedFiles, Optional.ofNullable(clientInfo));
+        builder.generateInsertRequest(requestId, pipe, files, showSkippedFiles);
 
     // send the request and get a response....
     try (CloseableHttpResponse response = httpClient.execute(httpPostForIngestFile)) {
-      LOGGER.info(
-          "Attempting to unmarshall insert response - {}, with clientInfo - {}",
-          response,
-          clientInfo);
+      LOGGER.info("Attempting to unmarshall insert response - {}", response);
       return ServiceResponseHandler.unmarshallIngestResponse(response, requestId);
     }
   }
@@ -635,52 +597,6 @@ public HistoryRangeResponse getHistoryRange(
     }
   }
 
-  /**
-   * Register a snowpipe client and returns the client sequencer
-   *
-   * @param requestId a UUID we use to label the request, if null, one is generated for the user
-   * @return
-   * @throws URISyntaxException - if the provided account name was illegal and caused a URI
-   *     construction failure
-   * @throws IOException - if we have some other network failure
-   * @throws IngestResponseException - if snowflake encountered error during ingest
-   * @throws BackOffException - if we have a 503 response
-   */
-  public ConfigureClientResponse configureClient(UUID requestId)
-      throws URISyntaxException, IOException, IngestResponseException, BackOffException {
-    if (requestId == null || requestId.toString().isEmpty()) {
-      requestId = UUID.randomUUID();
-    }
-    try (CloseableHttpResponse response =
-        httpClient.execute(builder.generateConfigureClientRequest(requestId, pipe))) {
-      LOGGER.info("Attempting to unmarshall configure client response - {}", response);
-      return ServiceResponseHandler.unmarshallConfigureClientResponse(response, requestId);
-    }
-  }
-
-  /**
-   * Get client status for snowpipe which contains offset token and client sequencer
-   *
-   * @param requestId a UUID we use to label the request, if null, one is generated for the user
-   * @return
-   * @throws URISyntaxException - if the provided account name was illegal and caused a URI
-   *     construction failure
-   * @throws IOException - if we have some other network failure
-   * @throws IngestResponseException - if snowflake encountered error during ingest
-   * @throws BackOffException - if we have a 503 response
-   */
-  public ClientStatusResponse getClientStatus(UUID requestId)
-      throws URISyntaxException, IOException, IngestResponseException, BackOffException {
-    if (requestId == null || requestId.toString().isEmpty()) {
-      requestId = UUID.randomUUID();
-    }
-    try (CloseableHttpResponse response =
-        httpClient.execute(builder.generateGetClientStatusRequest(requestId, pipe))) {
-      LOGGER.info("Attempting to unmarshall get client status response - {}", response);
-      return ServiceResponseHandler.unmarshallGetClientStatus(response, requestId);
-    }
-  }
-
   /**
    * Closes the resources associated with this object. Resources cannot be reopened, initialize new
    * instance of this class {@link SimpleIngestManager} to reopen and start ingesting/monitoring new
diff --git a/src/main/java/net/snowflake/ingest/connection/ClientStatusResponse.java b/src/main/java/net/snowflake/ingest/connection/ClientStatusResponse.java
deleted file mode 100644
index 5a3cbdc24..000000000
--- a/src/main/java/net/snowflake/ingest/connection/ClientStatusResponse.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright (c) 2021 Snowflake Computing Inc. All rights reserved.
- */
-package net.snowflake.ingest.connection;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-
-/** ClientStatusResponse - response from a get client status request */
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class ClientStatusResponse {
-  private Long clientSequencer;
-
-  private String offsetToken;
-
-  @Override
-  public String toString() {
-    return "IngestResponse{"
-        + "clientSequencer='"
-        + clientSequencer
-        + '\''
-        + ", offsetToken='"
-        + offsetToken
-        + '\''
-        + '}';
-  }
-
-  /**
-   * unique identifier for the client
-   *
-   * @return clientSequencer
-   */
-  public Long getClientSequencer() {
-    return clientSequencer;
-  }
-
-  /**
-   * offset number for kafka connector
-   *
-   * @return offsetToken
-   */
-  public String getOffsetToken() {
-    return offsetToken;
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/connection/ConfigureClientResponse.java b/src/main/java/net/snowflake/ingest/connection/ConfigureClientResponse.java
deleted file mode 100644
index 144065c22..000000000
--- a/src/main/java/net/snowflake/ingest/connection/ConfigureClientResponse.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright (c) 2021 Snowflake Computing Inc. All rights reserved.
- */
-package net.snowflake.ingest.connection;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-
-/** ConfigureClientResponse - response from a configure client request */
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class ConfigureClientResponse {
-  private Long clientSequencer;
-
-  @Override
-  public String toString() {
-    return "IngestResponse{" + "clientSequencer='" + clientSequencer + '\'' + '}';
-  }
-
-  /**
-   * unique identifier for the client
-   *
-   * @return clientSequencer
-   */
-  public Long getClientSequencer() {
-    return clientSequencer;
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/connection/InsertFilesClientInfo.java b/src/main/java/net/snowflake/ingest/connection/InsertFilesClientInfo.java
deleted file mode 100644
index 175f994d0..000000000
--- a/src/main/java/net/snowflake/ingest/connection/InsertFilesClientInfo.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright (c) 2021 Snowflake Computing Inc. All rights reserved.
- */
-package net.snowflake.ingest.connection;
-
-import java.nio.charset.StandardCharsets;
-
-/**
- * Just a wrapper class which is serialised into REST request for insertFiles.
- *
- * <p>This is an optional field which can be passed with a required field "files" in the request
- * body.
- *
- * <p>Here is how the new request could look like
- *
- * <pre>
- * {
- *   "files":[
- *     {
- *       "path":"file1.csv.gz"
- *     },
- *     {
- *       "path":"file2.csv.gz"
- *     }
- *    ],
- *   "clientInfo": {
- *     "clientSequencer": 1,
- *     "offsetToken": "2"
- *    }
- * }
- * </pre>
- */
-public class InsertFilesClientInfo {
-  //  FDB constant
-  public static final int MAX_ALLOWED_OFFSET_TOKEN_BYTE_SIZE = 100000;
-
-  // client sequencer which the caller thinks it currently has
-  private final long clientSequencer;
-
-  // offsetToken to atomically commit with files.
-  private final String offsetToken;
-
-  /** Constructor with both fields as required. */
-  public InsertFilesClientInfo(long clientSequencer, String offsetToken) {
-    if (clientSequencer < 0) {
-      throw new IllegalArgumentException("ClientSequencer should be non negative.");
-    }
-    if (offsetToken == null || offsetToken.isEmpty()) {
-      throw new IllegalArgumentException("OffsetToken can not be null or empty.");
-    }
-    /* Keeping some buffer */
-    if (offsetToken.getBytes(StandardCharsets.UTF_8).length
-        > MAX_ALLOWED_OFFSET_TOKEN_BYTE_SIZE - 1000) {
-      throw new IllegalArgumentException("OffsetToken size too large.");
-    }
-    this.clientSequencer = clientSequencer;
-    this.offsetToken = offsetToken;
-  }
-
-  /** Gets client Sequencer associated with this clientInfo record. */
-  public long getClientSequencer() {
-    return clientSequencer;
-  }
-
-  /** Gets offsetToken associated with this clientInfo record. */
-  public String getOffsetToken() {
-    return offsetToken;
-  }
-
-  /** Only printing clientSequencer */
-  @Override
-  public String toString() {
-    return "InsertFilesClientInfo{" + "clientSequencer=" + clientSequencer + '}';
-  }
-}
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 35bca3852..953e19009 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -7,7 +7,6 @@
 import static net.snowflake.ingest.utils.Constants.ENABLE_TELEMETRY_TO_SF;
 import static net.snowflake.ingest.utils.Utils.isNullOrEmpty;
 
-import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.net.URI;
@@ -15,7 +14,6 @@
 import java.security.KeyPair;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.UUID;
 import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders;
 import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpGet;
@@ -83,13 +81,6 @@ public class RequestBuilder {
   // the endpoint for history time range queries
   private static final String HISTORY_RANGE_ENDPOINT_FORMAT = "/v1/data/pipes/%s/loadHistoryScan";
 
-  // the endpoint for configure snowpipe client
-  private static final String CONFIGURE_CLIENT_ENDPOINT_FORMAT =
-      "/v1/data/pipes/%s/client/configure";
-
-  // the endpoint for get snowpipe client status
-  private static final String CLIENT_STATUS_ENDPOINT_FORMAT = "/v1/data/pipes/%s/client/status";
-
   // optional number of max seconds of items to fetch(eg. in the last hour)
   private static final String RECENT_HISTORY_IN_SECONDS = "recentSeconds";
 
@@ -332,35 +323,19 @@ private static class IngestRequest {
     // the list of files we're loading
     private final List<StagedFileWrapper> files;
 
-    // additional info passed along with files which includes clientSequencer and offsetToken
-    // can be null
-    @JsonInclude(JsonInclude.Include.NON_NULL)
-    private final InsertFilesClientInfo clientInfo;
-
-    /** Constructor used when both files and clientInfo are passed in request */
-    public IngestRequest(List<StagedFileWrapper> files, InsertFilesClientInfo clientInfo) {
-      this.files = files;
-      this.clientInfo = clientInfo;
-    }
-
     /**
      * Ctor used when only files is used in request body.
      *
      * <p>clientInfo will be defaulted to null
      */
     public IngestRequest(List<StagedFileWrapper> files) {
-      this(files, null);
+      this.files = files;
     }
 
     /* Gets the list of files which were added in request */
     public List<StagedFileWrapper> getFiles() {
       return files;
     }
-
-    /* Gets the clientInfo associated with files which was added in request */
-    public InsertFilesClientInfo getClientInfo() {
-      return clientInfo;
-    }
   }
 
   /**
@@ -501,79 +476,14 @@ private URI makeHistoryRangeURI(
   }
 
   /**
-   * Given a request UUID, and a fully qualified pipe name make a URI for configure snowpipe client
-   * http://snowflakeURL{:PORT}/v1/data/pipes/{pipeName}/client/configure
-   *
-   * <p>Where snowflake URL can be an old url or new regionless URL.
-   *
-   * <p>Request Body is Empty
-   *
-   * <p>And our response looks like:
-   *
-   * <pre>
-   *   {
-   *      'clientSequencer': LONG
-   *   }
-   * </pre>
-   *
-   * @param requestId the label for this request
-   * @param pipe the pipe name
-   * @return configure snowpipe client URI
-   * @throws URISyntaxException
-   */
-  private URI makeConfigureClientURI(UUID requestId, String pipe) throws URISyntaxException {
-    if (pipe == null) {
-      throw new IllegalArgumentException();
-    }
-    URIBuilder builder = makeBaseURI(requestId);
-    builder.setPath(String.format(CONFIGURE_CLIENT_ENDPOINT_FORMAT, pipe));
-    LOGGER.info("Final Configure Client URIBuilder - {}", builder);
-    return builder.build();
-  }
-
-  /**
-   * makeGetClientURI - Given a request UUID, and a fully qualified pipe name make a URI for getting
-   * snowpipe client http://snowflakeURL{:PORT}/v1/data/pipes/{pipeName}/client/status
-   *
-   * <p>Where snowflake URL can be an old url or new regionless URL.
-   *
-   * <p>Request Body is Empty
-   *
-   * <p>And our response looks like:
-   *
-   * <pre>
-   *   {
-   *      'offsetToken': STRING
-   *      'clientSequencer': LONG
-   *   }
-   * </pre>
-   *
-   * @param requestId the label for this request
-   * @param pipe the pipe name
-   * @return get client URI
-   * @throws URISyntaxException
-   */
-  private URI makeGetClientURI(UUID requestId, String pipe) throws URISyntaxException {
-    if (pipe == null) {
-      throw new IllegalArgumentException();
-    }
-    URIBuilder builder = makeBaseURI(requestId);
-    builder.setPath(String.format(CLIENT_STATUS_ENDPOINT_FORMAT, pipe));
-    LOGGER.info("Final Get Client URIBuilder - {}", builder);
-    return builder.build();
-  }
-
-  /**
-   * Given a list of files, and an optional clientInfo generate json string which later can be
-   * passed in request body of insertFiles API
+   * Given a list of files, generate a json string which later can be passed in request body of
+   * insertFiles API
    *
    * @param files the list of files we want to send
-   * @param clientInfo optional clientInfo which can be empty.
    * @return the string json blob
    * @throws IllegalArgumentException if files passed in is null
    */
-  private String serializeInsertFilesRequest(
-      List<StagedFileWrapper> files, Optional<InsertFilesClientInfo> clientInfo) {
+  private String serializeInsertFilesRequest(List<StagedFileWrapper> files) {
     // if the files argument is null, throw
     if (files == null) {
       LOGGER.info("Null files argument in RequestBuilder");
@@ -581,10 +491,7 @@ private String serializeInsertFilesRequest(
     }
 
     // create pojo
-    IngestRequest ingestRequest =
-        clientInfo
-            .map(insertFilesClientInfo -> new IngestRequest(files, insertFilesClientInfo))
-            .orElseGet(() -> new IngestRequest(files));
+    IngestRequest ingestRequest = new IngestRequest(files);
 
     // serialize to a string
     try {
@@ -648,28 +555,6 @@ private static void addHeaders(HttpUriRequest request, String token, String user
   public HttpPost generateInsertRequest(
       UUID requestId, String pipe, List<StagedFileWrapper> files, boolean showSkippedFiles)
       throws URISyntaxException {
-    return generateInsertRequest(requestId, pipe, files, showSkippedFiles, Optional.empty());
-  }
-
-  /**
-   * generateInsertRequest - given a pipe, list of files and clientInfo, make a request for the
-   * insert endpoint
-   *
-   * @param requestId a UUID we will use to label this request
-   * @param pipe a fully qualified pipe name
-   * @param files a list of files
-   * @param showSkippedFiles a boolean which returns skipped files when set to true
-   * @param clientInfo
-   * @return a post request with all the data we need
-   * @throws URISyntaxException if the URI components provided are improper
-   */
-  public HttpPost generateInsertRequest(
-      UUID requestId,
-      String pipe,
-      List<StagedFileWrapper> files,
-      boolean showSkippedFiles,
-      Optional<InsertFilesClientInfo> clientInfo)
-      throws URISyntaxException {
     // make the insert URI
     URI insertURI = makeInsertURI(requestId, pipe, showSkippedFiles);
     LOGGER.info("Created Insert Request : {} ", insertURI);
@@ -681,8 +566,7 @@ public HttpPost generateInsertRequest(
 
     // the entity for the containing the json
     final StringEntity entity =
-        new StringEntity(
-            serializeInsertFilesRequest(files, clientInfo), ContentType.APPLICATION_JSON);
+        new StringEntity(serializeInsertFilesRequest(files), ContentType.APPLICATION_JSON);
     post.setEntity(entity);
 
     return post;
@@ -769,22 +653,6 @@ public HttpPost generateStreamingIngestPostRequest(
     return post;
   }
 
-  /**
-   * Given a requestId and a pipe, make a configure client request
-   *
-   * @param requestID a UUID we will use to label this request
-   * @param pipe a fully qualified pipe name
-   * @return configure client request
-   * @throws URISyntaxException
-   */
-  public HttpPost generateConfigureClientRequest(UUID requestID, String pipe)
-      throws URISyntaxException {
-    URI configureClientURI = makeConfigureClientURI(requestID, pipe);
-    HttpPost post = new HttpPost(configureClientURI);
-    addHeaders(post, securityManager.getToken(), this.userAgentSuffix);
-    return post;
-  }
-
   /**
    * Generate post request for streaming ingest related APIs
    *
@@ -806,22 +674,6 @@ public HttpPost generateStreamingIngestPostRequest(
     return this.generateStreamingIngestPostRequest(payloadInString, endPoint, message);
   }
 
-  /**
-   * Given a requestId and a pipe, make a get client status request
-   *
-   * @param requestID UUID
-   * @param pipe a fully qualified pipe name
-   * @return get client status request
-   * @throws URISyntaxException
-   */
-  public HttpGet generateGetClientStatusRequest(UUID requestID, String pipe)
-      throws URISyntaxException {
-    URI getClientStatusURI = makeGetClientURI(requestID, pipe);
-    HttpGet get = new HttpGet(getClientStatusURI);
-    addHeaders(get, securityManager.getToken(), this.userAgentSuffix);
-    return get;
-  }
-
   /**
    * Closes the resources being used by RequestBuilder object. {@link SecurityManager} is one such
    * resource which uses a threadpool which needs to be shutdown once SimpleIngestManager is done
diff --git a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java
index f3da23991..807db49f3 100644
--- a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java
+++ b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java
@@ -34,8 +34,6 @@ public enum ApiName {
     INSERT_FILES("POST"),
     INSERT_REPORT("GET"),
     LOAD_HISTORY_SCAN("GET"),
-    CLIENT_CONFIGURE("POST"),
-    CLIENT_STATUS("GET"),
     STREAMING_OPEN_CHANNEL("POST"),
     STREAMING_CHANNEL_STATUS("POST"),
     STREAMING_REGISTER_BLOB("POST"),
@@ -154,64 +152,6 @@ public static HistoryRangeResponse unmarshallHistoryRangeResponse(
     return mapper.readValue(blob, HistoryRangeResponse.class);
   }
 
-  /**
-   * unmarshallConfigureClientResponse - Given an HttpResponse object, attempts to deserialize it
-   * into a ConfigureClientResponse
-   *
-   * @param response HttpResponse
-   * @param requestId
-   * @return ConfigureClientResponse
-   * @throws IOException if our entity is somehow corrupt or we can't get it
-   * @throws IngestResponseException - if we have an uncategorized network issue
-   * @throws BackOffException - if we have a 503 issue
-   */
-  public static ConfigureClientResponse unmarshallConfigureClientResponse(
-      HttpResponse response, UUID requestId)
-      throws IOException, IngestResponseException, BackOffException {
-    if (response == null) {
-      LOGGER.warn("Null response passed to unmarshallConfigureClientResponse");
-      throw new IllegalArgumentException();
-    }
-
-    // handle the exceptional status code
-    handleExceptionalStatus(response, requestId, ApiName.CLIENT_CONFIGURE);
-
-    // grab the string version of the response entity
-    String blob = consumeAndReturnResponseEntityAsString(response.getEntity());
-
-    // read out our blob into a pojo
-    return mapper.readValue(blob, ConfigureClientResponse.class);
-  }
-
-  /**
-   * unmarshallGetClientStatus - Given an HttpResponse object, attempts to deserialize it into a
-   * ClientStatusResponse
-   *
-   * @param response HttpResponse
-   * @param requestId
-   * @return ClientStatusResponse
-   * @throws IOException if our entity is somehow corrupt or we can't get it
-   * @throws IngestResponseException - if we have an uncategorized network issue
-   * @throws BackOffException - if we have a 503 issue
-   */
-  public static ClientStatusResponse unmarshallGetClientStatus(
-      HttpResponse response, UUID requestId)
-      throws IOException, IngestResponseException, BackOffException {
-    if (response == null) {
-      LOGGER.warn("Null response passed to unmarshallClientStatusResponse");
-      throw new IllegalArgumentException();
-    }
-
-    // handle the exceptional status code
-    handleExceptionalStatus(response, requestId, ApiName.CLIENT_STATUS);
-
-    // grab the string version of the response entity
-    String blob = consumeAndReturnResponseEntityAsString(response.getEntity());
-
-    // read out our blob into a pojo
-    return mapper.readValue(blob, ClientStatusResponse.class);
-  }
-
   /**
    * unmarshallStreamingIngestResponse Given an HttpResponse object - attempts to deserialize it
    * into an Object based on input type
diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
index b8740ee2e..f1c4c06f1 100644
--- a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
@@ -5,8 +5,6 @@
 import static net.snowflake.ingest.connection.RequestBuilder.JAVA_USER_AGENT;
 import static net.snowflake.ingest.connection.RequestBuilder.OS_INFO_USER_AGENT_FORMAT;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -23,17 +21,11 @@
 import net.snowflake.client.jdbc.internal.apache.http.Header;
 import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders;
 import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost;
-import net.snowflake.ingest.connection.ClientStatusResponse;
-import net.snowflake.ingest.connection.ConfigureClientResponse;
 import net.snowflake.ingest.connection.HistoryResponse;
 import net.snowflake.ingest.connection.IngestResponse;
-import net.snowflake.ingest.connection.IngestResponseException;
-import net.snowflake.ingest.connection.InsertFilesClientInfo;
 import net.snowflake.ingest.utils.StagedFileWrapper;
 import org.junit.After;
-import org.junit.Assert;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 
 /** Example ingest sdk integration test */
@@ -383,181 +375,4 @@ private void verifyDefaultUserAgent(
       }
     }
   }
-
-  @Test
-  @Ignore("SNOW-693813") // Disable Snowpipe Exactly once, remove later
-  public void testConfigureClientHappyCase() throws Exception {
-    final String userAgentSuffix = "kafka-provider/NONE";
-    SimpleIngestManager manager = TestUtils.getManager(pipeName, userAgentSuffix);
-    ConfigureClientResponse configureClientResponse = manager.configureClient(null);
-    assertEquals(0L, configureClientResponse.getClientSequencer().longValue());
-  }
-
-  @Test
-  @Ignore("SNOW-693813") // Disable Snowpipe Exactly once, remove later
-  public void testConfigureClientNoPipeFound() throws Exception {
-    final String userAgentSuffix = "kafka-provider/NONE";
-    SimpleIngestManager manager = TestUtils.getManager("nopipe", userAgentSuffix);
-    try {
-      manager.configureClient(null);
-    } catch (IngestResponseException exception) {
-      assertEquals(404, exception.getErrorCode());
-      assertEquals(
-          "Specified object does not exist or not authorized. Pipe not found",
-          exception.getErrorBody().getMessage());
-    }
-  }
-
-  @Test
-  @Ignore("SNOW-693813") // Disable Snowpipe Exactly once, remove later
-  public void testGetClientStatusHappyCase() throws Exception {
-    final String userAgentSuffix = "kafka-provider/NONE";
-    SimpleIngestManager manager = TestUtils.getManager(pipeName, userAgentSuffix);
-    manager.configureClient(null);
-    ClientStatusResponse clientStatusResponse = manager.getClientStatus(null);
-    assertEquals(0L, clientStatusResponse.getClientSequencer().longValue());
-    assertNull(clientStatusResponse.getOffsetToken());
-  }
-
-  @Test
-  @Ignore("SNOW-693813") // Disable Snowpipe Exactly once, remove later
-  public void testGetClientStatusNoPipeFound() throws Exception {
-    final String userAgentSuffix = "kafka-provider/NONE";
-    SimpleIngestManager manager = TestUtils.getManager("nopipe", userAgentSuffix);
-    try {
-      manager.getClientStatus(null);
-    } catch (IngestResponseException exception) {
-      assertEquals(404, exception.getErrorCode());
-      assertEquals(
-          "Specified object does not exist or not authorized. Pipe not found",
-          exception.getErrorBody().getMessage());
-    }
-  }
-
-  @Test
-  @Ignore("SNOW-693813") // Disable Snowpipe Exactly once, remove later
-  public void testIngestFilesWithClientInfo() throws Exception {
-
-    // first lets call configure client API
-    final String userAgentSuffix = "kafka-provider/NONE";
-    SimpleIngestManager manager = TestUtils.getManager(pipeName, userAgentSuffix);
-    ConfigureClientResponse configureClientResponse = manager.configureClient(null);
-    assertEquals(0L, configureClientResponse.getClientSequencer().longValue());
-
-    // put
-    TestUtils.executeQuery("put file://" + testFilePath + " @" + stageName);
-
-    // create a file wrapper
-    StagedFileWrapper myFile = new StagedFileWrapper(TEST_FILE_NAME, null);
-
-    final String offsetToken = "1";
-    InsertFilesClientInfo clientInfo =
-        new InsertFilesClientInfo(configureClientResponse.getClientSequencer(), offsetToken);
-
-    // get an insert response after we submit
-    IngestResponse insertResponse =
-        manager.ingestFiles(Collections.singletonList(myFile), null, false, clientInfo);
-
-    assertEquals("SUCCESS", insertResponse.getResponseCode());
-
-    // Get history and ensure that the expected file has been ingested
-    getHistoryAndAssertLoad(manager, TEST_FILE_NAME);
-
-    // Get client status since we added offsetToken too
-    ClientStatusResponse clientStatusResponse = manager.getClientStatus(null);
-    assertEquals(0L, clientStatusResponse.getClientSequencer().longValue());
-    assertNotNull(clientStatusResponse.getOffsetToken());
-    assertEquals(offsetToken, clientStatusResponse.getOffsetToken());
-  }
-
-  @Test
-  @Ignore("SNOW-693813") // Disable Snowpipe Exactly once, remove later
-  public void testIngestFilesWithClientInfoWithOldClientSequencer() throws Exception {
-
-    // first lets call configure client API
-    final String userAgentSuffix = "kafka-provider/NONE";
-    SimpleIngestManager manager = TestUtils.getManager(pipeName, userAgentSuffix);
-    ConfigureClientResponse configureClientResponse = manager.configureClient(null);
-    assertEquals(0L, configureClientResponse.getClientSequencer().longValue());
-    final long oldClientSequencer = configureClientResponse.getClientSequencer();
-    configureClientResponse = manager.configureClient(null);
-    assertEquals(1L, configureClientResponse.getClientSequencer().longValue());
-
-    // put
-    TestUtils.executeQuery("put file://" + testFilePath + " @" + stageName);
-
-    // create a file wrapper
-    StagedFileWrapper myFile = new StagedFileWrapper(TEST_FILE_NAME, null);
-
-    final String offsetToken = "1";
-    // Passing in an old clientSequencer
-    InsertFilesClientInfo clientInfo = new InsertFilesClientInfo(oldClientSequencer, offsetToken);
-
-    // get an insert response after we submit
-    try {
-      manager.ingestFiles(Collections.singletonList(myFile), null, false, clientInfo);
-      Assert.fail(
-          "The insertFiles API should return 400 and SDK should throw IngestResponseException");
-    } catch (IngestResponseException ex) {
-      assertEquals(400, ex.getErrorCode());
-      assertTrue(ex.getErrorBody().getCode().equalsIgnoreCase("091129"));
-    }
-
-    // Get client status since we added offsetToken too
-    ClientStatusResponse clientStatusResponse = manager.getClientStatus(null);
-    assertEquals(
-        configureClientResponse.getClientSequencer(), clientStatusResponse.getClientSequencer());
-    assertNull(clientStatusResponse.getOffsetToken());
-
-    // lets call insertFiles with new clientSequencer
-    // Passing in a new clientSequencer
-    clientInfo = new InsertFilesClientInfo(clientStatusResponse.getClientSequencer(), offsetToken);
-
-    // get an insert response after we submit
-    try {
-      IngestResponse insertResponse =
-          manager.ingestFiles(Collections.singletonList(myFile), null, false, clientInfo);
-      assertEquals("SUCCESS", insertResponse.getResponseCode());
-
-      // Get history and ensure that the expected file has been ingested
-      getHistoryAndAssertLoad(manager, TEST_FILE_NAME);
-
-      // Get client status since we added offsetToken too (During second attempt)
-      clientStatusResponse = manager.getClientStatus(null);
-      assertNotNull(clientStatusResponse.getOffsetToken());
-      assertEquals(offsetToken, clientStatusResponse.getOffsetToken());
-    } catch (IngestResponseException ex) {
-      Assert.fail(
-          "The insertFiles API should be successful second time after updating clientSequencer");
-    }
-  }
-
-  @Test
-  @Ignore("SNOW-693813") // Disable Snowpipe Exactly once, remove later
-  public void testIngestFilesWithClientInfoWithNoClientSequencer() throws Exception {
-    // first lets call configure client API
-    final String userAgentSuffix = "kafka-provider/NONE";
-    SimpleIngestManager manager = TestUtils.getManager(pipeName, userAgentSuffix);
-
-    // put
-    TestUtils.executeQuery("put file://" + testFilePath + " @" + stageName);
-
-    // create a file wrapper
-    StagedFileWrapper myFile = new StagedFileWrapper(TEST_FILE_NAME, null);
-
-    final String offsetToken = "1";
-    // Passing in an old clientSequencer
-    InsertFilesClientInfo clientInfo = new InsertFilesClientInfo(0L, offsetToken);
-
-    // get an insert response after we submit
-    try {
-      manager.ingestFiles(Collections.singletonList(myFile), null, false, clientInfo);
-      Assert.fail(
-          "The insertFiles API should return 400 since client/configure was not called and SDK"
-              + " should throw IngestResponseException");
-    } catch (IngestResponseException ex) {
-      assertEquals(400, ex.getErrorCode());
-      assertTrue(ex.getErrorBody().getCode().equalsIgnoreCase("091128"));
-    }
-  }
 }

From 70ac930f4e6372d001c6ef2b63ce8bb44e62c1b9 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Thu, 6 Jul 2023 12:58:44 +0200
Subject: [PATCH 264/356] SNOW-855191 Race condition causing NPE (#544)

There is a race condition, which leads to a NPE when one thread invalidates the channel,
while another thread is inside of insertRows(). The problem is that channel invalidation
clears fieldIndex, which may still be used in threads concurrently inserting rows.

There is no need to clear the fieldIndex during invalidation, it is enough to mark
the channel as invalid and it won't be flushed. When channel is reopened, setupSchema()
will clear fieldIndex and reinitialize the schema.
---
 .../streaming/internal/ParquetRowBuffer.java  |  1 -
 .../streaming/internal/RowBufferTest.java     | 19 -------------------
 2 files changed, 20 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index baf7fa3bc..99fd25fb0 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -309,7 +309,6 @@ void reset() {
   /** Close the row buffer by releasing its internal resources. */
   @Override
   void closeInternal() {
-    this.fieldIndex.clear();
     if (bdecParquetWriter != null) {
       try {
         bdecParquetWriter.close();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 321d76c93..7fa598647 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -420,25 +420,6 @@ private void testInsertRowsHelper(AbstractRowBuffer<?> rowBuffer) {
     Assert.assertFalse(response.hasErrors());
   }
 
-  @Test
-  public void testClose() {
-    this.rowBufferOnErrorContinue.close("testClose");
-    Map<String, Object> row = new HashMap<>();
-    row.put("colTinyInt", (byte) 1);
-    row.put("colSmallInt", (short) 2);
-    row.put("colInt", 3);
-    row.put("colBigInt", 4L);
-    row.put("colDecimal", 1.23);
-    row.put("colChar", "2");
-
-    try {
-      this.rowBufferOnErrorContinue.insertRows(Collections.singletonList(row), null);
-      Assert.fail("Insert should fail after buffer is closed");
-    } catch (SFException e) {
-      Assert.assertEquals(ErrorCode.INTERNAL_ERROR.getMessageCode(), e.getVendorCode());
-    }
-  }
-
   @Test
   public void testFlush() {
     testFlushHelper(this.rowBufferOnErrorAbort);

From 88e9dd1a5ed9ed6367241193bfec8184609fed73 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Fri, 7 Jul 2023 10:46:51 +0200
Subject: [PATCH 265/356] SNOW-851351 Retry javax.net.ssl.SSLException (#543)

This PR retries SSLException, which we have been seeing in client logs.
It additionally evicts idle connections after 60s of inactivity.
---
 .../net/snowflake/ingest/utils/HttpUtil.java  | 26 +++++++++---
 .../snowflake/ingest/utils/HttpUtilTest.java  | 41 +++++++++++++++++++
 2 files changed, 61 insertions(+), 6 deletions(-)
 create mode 100644 src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java

diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index 72bf32767..fcec12392 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -73,6 +73,12 @@ public class HttpUtil {
   private static final int DEFAULT_CONNECTION_TIMEOUT_MINUTES = 1;
   private static final int DEFAULT_HTTP_CLIENT_SOCKET_TIMEOUT_MINUTES = 5;
 
+  /**
+   * After how many seconds of inactivity should be idle connections evicted from the connection
+   * pool.
+   */
+  private static final int DEFAULT_EVICT_IDLE_AFTER_SECONDS = 60;
+
   // Default is 2, but scaling it up to 100 to match with default_max_connections
   private static final int DEFAULT_MAX_CONNECTIONS_PER_ROUTE = 100;
 
@@ -146,6 +152,7 @@ private static void initHttpClient(String accountName) {
     HttpClientBuilder clientBuilder =
         HttpClientBuilder.create()
             .setConnectionManager(connectionManager)
+            .evictIdleConnections(DEFAULT_EVICT_IDLE_AFTER_SECONDS, TimeUnit.SECONDS)
             .setSSLSocketFactory(f)
             .setServiceUnavailableRetryStrategy(getServiceUnavailableRetryStrategy())
             .setRetryHandler(getHttpRequestRetryHandler())
@@ -259,27 +266,34 @@ public long getRetryInterval() {
   }
 
   /**
-   * Retry handler logic. Retry if No response from Service exception. (NoHttpResponseException)
+   * Retry handler logic. Retry at most {@link HttpUtil.MAX_RETRIES} times if any of the following
+   * exceptions is thrown by request execution:
+   *
+   * <ul>
+   *   <li>No response from Service exception (NoHttpResponseException)
+   *   <li>javax.net.ssl.SSLException: Connection reset.
+   * </ul>
    *
    * @return retryHandler to add to http client.
    */
-  private static HttpRequestRetryHandler getHttpRequestRetryHandler() {
+  static HttpRequestRetryHandler getHttpRequestRetryHandler() {
     return (exception, executionCount, httpContext) -> {
       final String requestURI = getRequestUriFromContext(httpContext);
       if (executionCount > MAX_RETRIES) {
         LOGGER.info("Max retry exceeded for requestURI:{}", requestURI);
         return false;
       }
-      if (exception instanceof NoHttpResponseException) {
+      if (exception instanceof NoHttpResponseException
+          || exception instanceof javax.net.ssl.SSLException) {
         LOGGER.info(
-            "Retrying request which caused No HttpResponse Exception with "
-                + "URI:{}, retryCount:{} and maxRetryCount:{}",
+            "Retrying request which caused {} with " + "URI:{}, retryCount:{} and maxRetryCount:{}",
+            exception.getClass().getName(),
             requestURI,
             executionCount,
             MAX_RETRIES);
         return true;
       }
-      LOGGER.info("No retry for URI:{} with exception", requestURI, exception);
+      LOGGER.info("No retry for URI:{} with exception {}", requestURI, exception.toString());
       return false;
     };
   }
diff --git a/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java b/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java
new file mode 100644
index 000000000..5e21cad99
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java
@@ -0,0 +1,41 @@
+package net.snowflake.ingest.utils;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.doReturn;
+
+import java.io.IOException;
+import javax.net.ssl.SSLException;
+import net.snowflake.client.jdbc.internal.apache.http.HttpRequest;
+import net.snowflake.client.jdbc.internal.apache.http.NoHttpResponseException;
+import net.snowflake.client.jdbc.internal.apache.http.RequestLine;
+import net.snowflake.client.jdbc.internal.apache.http.client.HttpRequestRetryHandler;
+import net.snowflake.client.jdbc.internal.apache.http.client.protocol.HttpClientContext;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+public class HttpUtilTest {
+  @Test
+  public void testRequestRetryHandler() {
+    HttpRequestRetryHandler httpRequestRetryHandler = HttpUtil.getHttpRequestRetryHandler();
+
+    HttpClientContext httpContextMock = Mockito.mock(HttpClientContext.class);
+    RequestLine requestLine = Mockito.mock(RequestLine.class);
+    HttpRequest httpRequest = Mockito.mock(HttpRequest.class);
+
+    doReturn(httpRequest).when(httpContextMock).getRequest();
+    doReturn(requestLine).when(httpRequest).getRequestLine();
+    doReturn("/api/v1/status").when(requestLine).getUri();
+
+    assertTrue(
+        httpRequestRetryHandler.retryRequest(
+            new NoHttpResponseException("Test exception"), 1, httpContextMock));
+    assertTrue(
+        httpRequestRetryHandler.retryRequest(
+            new SSLException("Test exception"), 1, httpContextMock));
+    assertFalse(
+        httpRequestRetryHandler.retryRequest(
+            new SSLException("Test exception"), 4, httpContextMock));
+    assertFalse(httpRequestRetryHandler.retryRequest(new IOException(), 1, httpContextMock));
+  }
+}

From 51a6f0915d454dfd402cb399dee89539cbad2c07 Mon Sep 17 00:00:00 2001
From: Alec Huang <alec.huang@snowflake.com>
Date: Wed, 12 Jul 2023 14:54:15 -0700
Subject: [PATCH 266/356] SNOW-352846 OAuth Authentication: #2 OAuth Support
 (#537)

---
 .../ingest/connection/JWTManager.java         |   2 +-
 .../ingest/connection/OAuthClient.java        |  14 ++
 .../ingest/connection/OAuthCredential.java    |  55 ++++++
 .../ingest/connection/OAuthManager.java       | 175 ++++++++++++++++++
 .../ingest/connection/RequestBuilder.java     | 131 +++++++++----
 .../connection/SnowflakeOAuthClient.java      | 138 ++++++++++++++
 .../SnowflakeStreamingIngestClient.java       |   8 +
 ...nowflakeStreamingIngestClientInternal.java |  55 ++++--
 .../net/snowflake/ingest/utils/Constants.java |   7 +
 .../net/snowflake/ingest/utils/ErrorCode.java |   5 +-
 .../net/snowflake/ingest/utils/Utils.java     |  26 ++-
 .../java/net/snowflake/ingest/TestUtils.java  |  27 +++
 .../ingest/connection/MockOAuthClient.java    |  39 ++++
 .../connection/SecurityManagerTest.java       | 112 ++++++++---
 .../streaming/internal/OAuthBasicTest.java    | 121 ++++++++++++
 15 files changed, 835 insertions(+), 80 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/connection/OAuthClient.java
 create mode 100644 src/main/java/net/snowflake/ingest/connection/OAuthCredential.java
 create mode 100644 src/main/java/net/snowflake/ingest/connection/OAuthManager.java
 create mode 100644 src/main/java/net/snowflake/ingest/connection/SnowflakeOAuthClient.java
 create mode 100644 src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/OAuthBasicTest.java

diff --git a/src/main/java/net/snowflake/ingest/connection/JWTManager.java b/src/main/java/net/snowflake/ingest/connection/JWTManager.java
index e5ee3aa0b..1484a3ce9 100644
--- a/src/main/java/net/snowflake/ingest/connection/JWTManager.java
+++ b/src/main/java/net/snowflake/ingest/connection/JWTManager.java
@@ -35,7 +35,7 @@ final class JWTManager extends SecurityManager {
   private final transient KeyPair keyPair;
 
   // the token itself
-  protected final AtomicReference<String> token;
+  private final AtomicReference<String> token;
 
   /**
    * Creates a JWTManager entity for a given account, user and KeyPair with a specified time to
diff --git a/src/main/java/net/snowflake/ingest/connection/OAuthClient.java b/src/main/java/net/snowflake/ingest/connection/OAuthClient.java
new file mode 100644
index 000000000..bdff6c96e
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/connection/OAuthClient.java
@@ -0,0 +1,14 @@
+/*
+ * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.connection;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+/** Interface to perform token refresh request from {@link OAuthManager} */
+public interface OAuthClient {
+  AtomicReference<OAuthCredential> getoAuthCredentialRef();
+
+  void refreshToken();
+}
diff --git a/src/main/java/net/snowflake/ingest/connection/OAuthCredential.java b/src/main/java/net/snowflake/ingest/connection/OAuthCredential.java
new file mode 100644
index 000000000..e222478d1
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/connection/OAuthCredential.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.connection;
+
+import java.util.Base64;
+
+/** This class hold credentials for OAuth authentication */
+public class OAuthCredential {
+  private static final String BASIC_AUTH_HEADER_PREFIX = "Basic ";
+  private final String authHeader;
+  private final String clientId;
+  private final String clientSecret;
+  private String accessToken;
+  private String refreshToken;
+  private int expiresIn;
+
+  public OAuthCredential(String clientId, String clientSecret, String refreshToken) {
+    this.authHeader =
+        BASIC_AUTH_HEADER_PREFIX
+            + Base64.getEncoder().encodeToString((clientId + ":" + clientSecret).getBytes());
+    this.clientId = clientId;
+    this.clientSecret = clientSecret;
+    this.refreshToken = refreshToken;
+  }
+
+  public String getAuthHeader() {
+    return authHeader;
+  }
+
+  public String getAccessToken() {
+    return accessToken;
+  }
+
+  public void setAccessToken(String accessToken) {
+    this.accessToken = accessToken;
+  }
+
+  public String getRefreshToken() {
+    return refreshToken;
+  }
+
+  public void setRefreshToken(String refreshToken) {
+    this.refreshToken = refreshToken;
+  }
+
+  public void setExpiresIn(int expiresIn) {
+    this.expiresIn = expiresIn;
+  }
+
+  public int getExpiresIn() {
+    return expiresIn;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/connection/OAuthManager.java b/src/main/java/net/snowflake/ingest/connection/OAuthManager.java
new file mode 100644
index 000000000..8ba2c05a0
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/connection/OAuthManager.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.connection;
+
+import java.util.concurrent.TimeUnit;
+import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.SFException;
+
+/** This class manages creating and automatically refresh the OAuth token */
+public final class OAuthManager extends SecurityManager {
+  private static final double DEFAULT_UPDATE_THRESHOLD_RATIO = 0.8;
+
+  // the endpoint for token request
+  private static final String TOKEN_REQUEST_ENDPOINT = "/oauth/token-request";
+
+  // Content type header to specify the encoding
+  private static final String OAUTH_CONTENT_TYPE_HEADER = "application/x-www-form-urlencoded";
+
+  // Properties for token refresh request
+  private static final String TOKEN_TYPE = "OAUTH";
+
+  // Update threshold, a floating-point value representing the ratio between the expiration time of
+  // an access token and the time needed to update it. It must be a value greater than 0 and less
+  // than 1. E.g. An access token with expires_in=600 and update_threshold_ratio=0.8 would be
+  // updated after 600*0.8 = 480.
+  private final double updateThresholdRatio;
+
+  private OAuthClient oAuthClient;
+
+  /**
+   * Creates a OAuthManager entity for a given account, user and OAuthCredential with default time
+   * to refresh the access token
+   *
+   * @param accountName - the snowflake account name of this user
+   * @param username - the snowflake username of the current user
+   * @param oAuthCredential - the OAuth credential we're using to connect
+   * @param baseURIBuilder - the uri builder with common scheme, host and port
+   * @param telemetryService reference to the telemetry service
+   */
+  OAuthManager(
+      String accountName,
+      String username,
+      OAuthCredential oAuthCredential,
+      URIBuilder baseURIBuilder,
+      TelemetryService telemetryService) {
+    this(
+        accountName,
+        username,
+        oAuthCredential,
+        baseURIBuilder,
+        DEFAULT_UPDATE_THRESHOLD_RATIO,
+        telemetryService);
+  }
+
+  /**
+   * Creates a OAuthManager entity for a given account, user and OAuthCredential with a specified
+   * time to refresh the token
+   *
+   * @param accountName - the snowflake account name of this user
+   * @param username - the snowflake username of the current user
+   * @param oAuthCredential - the OAuth credential we're using to connect
+   * @param baseURIBuilder - the uri builder with common scheme, host and port
+   * @param updateThresholdRatio - the ratio between the expiration time of a token and the time
+   *     needed to refresh it.
+   * @param telemetryService reference to the telemetry service
+   */
+  OAuthManager(
+      String accountName,
+      String username,
+      OAuthCredential oAuthCredential,
+      URIBuilder baseURIBuilder,
+      double updateThresholdRatio,
+      TelemetryService telemetryService) {
+    // disable telemetry service until jdbc v3.13.34 is released
+    super(accountName, username, null);
+
+    // if any of our arguments are null, throw an exception
+    if (oAuthCredential == null || baseURIBuilder == null) {
+      throw new IllegalArgumentException();
+    }
+
+    // check if update threshold is within (0, 1)
+    if (updateThresholdRatio <= 0 || updateThresholdRatio >= 1) {
+      throw new IllegalArgumentException("updateThresholdRatio should fall in (0, 1)");
+    }
+    this.updateThresholdRatio = updateThresholdRatio;
+    this.oAuthClient = new SnowflakeOAuthClient(accountName, oAuthCredential, baseURIBuilder);
+
+    // generate our first token
+    refreshToken();
+  }
+
+  /**
+   * Creates a OAuthManager entity for a given account, user and OAuthClient with a specified time
+   * to refresh the token. Use for testing only.
+   *
+   * @param accountName - the snowflake account name of this user
+   * @param username - the snowflake username of the current user
+   * @param oAuthClient - the OAuth client to perform token refresh
+   * @param updateThresholdRatio - the ratio between the expiration time of a token and the time
+   *     needed to refresh it.
+   */
+  public OAuthManager(
+      String accountName, String username, OAuthClient oAuthClient, double updateThresholdRatio) {
+    super(accountName, username, null);
+
+    this.updateThresholdRatio = updateThresholdRatio;
+    this.oAuthClient = oAuthClient;
+
+    refreshToken();
+  }
+
+  @Override
+  String getToken() {
+    if (refreshFailed.get()) {
+      throw new SecurityException("getToken request failed due to token refresh failure");
+    }
+    return oAuthClient.getoAuthCredentialRef().get().getAccessToken();
+  }
+
+  @Override
+  String getTokenType() {
+    return TOKEN_TYPE;
+  }
+
+  /**
+   * Set refresh token, this method is for refresh token renewal without requiring to restart
+   * client. This method only works when the authorization type is OAuth.
+   *
+   * @param refreshToken the new refresh token
+   */
+  void setRefreshToken(String refreshToken) {
+    oAuthClient.getoAuthCredentialRef().get().setRefreshToken(refreshToken);
+  }
+
+  /** refreshToken - Get new access token using refresh_token, client_id, client_secret */
+  private void refreshToken() {
+    for (int retries = 0; retries < Constants.MAX_OAUTH_REFRESH_TOKEN_RETRY; retries++) {
+      try {
+        oAuthClient.refreshToken();
+
+        // Schedule next refresh
+        long nextRefreshDelay =
+            (long)
+                (oAuthClient.getoAuthCredentialRef().get().getExpiresIn()
+                    * this.updateThresholdRatio);
+        tokenRefresher.schedule(this::refreshToken, nextRefreshDelay, TimeUnit.SECONDS);
+        LOGGER.debug(
+            "Refresh access token, next refresh is scheduled after {} seconds", nextRefreshDelay);
+
+        return;
+      } catch (SFException e1) {
+        // Exponential backoff retries
+        try {
+          Thread.sleep((1L << retries) * 1000L);
+        } catch (InterruptedException e2) {
+          throw new SFException(ErrorCode.OAUTH_REFRESH_TOKEN_ERROR, e2.getMessage());
+        }
+      }
+    }
+
+    refreshFailed.set(true);
+    throw new SecurityException("Fail to refresh access token");
+  }
+
+  /** Currently, it only shuts down the instance of ExecutorService. */
+  @Override
+  public void close() {
+    tokenRefresher.shutdown();
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 953e19009..0aecf1a67 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012-2017 Snowflake Computing Inc. All rights reserved.
+ * Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved.
  */
 
 package net.snowflake.ingest.connection;
@@ -128,10 +128,11 @@ public class RequestBuilder {
    *
    * @param accountName - the name of the Snowflake account to which we're connecting
    * @param userName - the username of the entity loading files
-   * @param keyPair - the Public/Private key pair we'll use to authenticate
+   * @param credential - the credential we'll use to authenticate
    */
-  public RequestBuilder(String accountName, String userName, KeyPair keyPair) {
-    this(accountName, userName, keyPair, DEFAULT_SCHEME, DEFAULT_HOST_SUFFIX, DEFAULT_PORT, null);
+  public RequestBuilder(String accountName, String userName, Object credential) {
+    this(
+        accountName, userName, credential, DEFAULT_SCHEME, DEFAULT_HOST_SUFFIX, DEFAULT_PORT, null);
   }
 
   /**
@@ -139,16 +140,17 @@ public RequestBuilder(String accountName, String userName, KeyPair keyPair) {
    *
    * @param accountName - the name of the Snowflake account to which we're connecting
    * @param userName - the username of the entity loading files
-   * @param keyPair - the Public/Private key pair we'll use to authenticate
+   * @param credential - the credential we'll use to authenticate
    * @param userAgentSuffix - The suffix part of HTTP Header User-Agent
    */
   public RequestBuilder(
       String accountName,
       String userName,
       String hostName,
-      KeyPair keyPair,
+      Object credential,
       String userAgentSuffix) {
-    this(accountName, userName, keyPair, DEFAULT_SCHEME, hostName, DEFAULT_PORT, userAgentSuffix);
+    this(
+        accountName, userName, credential, DEFAULT_SCHEME, hostName, DEFAULT_PORT, userAgentSuffix);
   }
 
   /**
@@ -157,7 +159,7 @@ public RequestBuilder(
    *
    * @param accountName - the account name to which we're connecting
    * @param userName - for whom are we connecting?
-   * @param keyPair - our auth credentials
+   * @param credential - the credential we'll use to authenticate
    * @param schemeName - are we HTTP or HTTPS?
    * @param hostName - the host for this snowflake instance
    * @param portNum - the port number
@@ -165,11 +167,11 @@ public RequestBuilder(
   public RequestBuilder(
       String accountName,
       String userName,
-      KeyPair keyPair,
+      Object credential,
       String schemeName,
       String hostName,
       int portNum) {
-    this(accountName, userName, keyPair, schemeName, hostName, portNum, null);
+    this(accountName, userName, credential, schemeName, hostName, portNum, null);
   }
 
   /**
@@ -177,7 +179,7 @@ public RequestBuilder(
    *
    * @param accountName - the account name to which we're connecting
    * @param userName - for whom are we connecting?
-   * @param keyPair - our auth credentials
+   * @param credential - the credential we'll use to authenticate
    * @param schemeName - are we HTTP or HTTPS?
    * @param hostName - the host for this snowflake instance
    * @param portNum - the port number
@@ -186,13 +188,22 @@ public RequestBuilder(
   public RequestBuilder(
       String accountName,
       String userName,
-      KeyPair keyPair,
+      Object credential,
       String schemeName,
       String hostName,
       int portNum,
       String userAgentSuffix) {
     this(
-        accountName, userName, keyPair, schemeName, hostName, portNum, userAgentSuffix, null, null);
+        accountName,
+        userName,
+        credential,
+        schemeName,
+        hostName,
+        portNum,
+        userAgentSuffix,
+        null,
+        null,
+        null);
   }
 
   /**
@@ -200,24 +211,25 @@ public RequestBuilder(
    *
    * @param url - the Snowflake account to which we're connecting
    * @param userName - the username of the entity loading files
-   * @param keyPair - the Public/Private key pair we'll use to authenticate
+   * @param credential - the credential we'll use to authenticate
    * @param httpClient - reference to the http client
    * @param clientName - name of the client, used to uniquely identify a client if used
    */
   public RequestBuilder(
       SnowflakeURL url,
       String userName,
-      KeyPair keyPair,
+      Object credential,
       CloseableHttpClient httpClient,
       String clientName) {
     this(
         url.getAccount(),
         userName,
-        keyPair,
+        credential,
         url.getScheme(),
         url.getUrlWithoutPort(),
         url.getPort(),
         null,
+        null,
         httpClient,
         clientName);
   }
@@ -227,40 +239,40 @@ public RequestBuilder(
    *
    * @param accountName - the account name to which we're connecting
    * @param userName - for whom are we connecting?
-   * @param keyPair - our auth credentials
+   * @param credential - our auth credentials, either JWT key pair or OAuth credential
    * @param schemeName - are we HTTP or HTTPS?
    * @param hostName - the host for this snowflake instance
    * @param portNum - the port number
    * @param userAgentSuffix - The suffix part of HTTP Header User-Agent
+   * @param securityManager - The security manager for authentication
    * @param httpClient - reference to the http client
    * @param clientName - name of the client, used to uniquely identify a client if used
    */
   public RequestBuilder(
       String accountName,
       String userName,
-      KeyPair keyPair,
+      Object credential,
       String schemeName,
       String hostName,
       int portNum,
       String userAgentSuffix,
+      SecurityManager securityManager,
       CloseableHttpClient httpClient,
       String clientName) {
     // none of these arguments should be null
-    if (accountName == null || userName == null || keyPair == null) {
+    if (accountName == null || userName == null || credential == null) {
       throw new IllegalArgumentException();
     }
 
     // Set up the telemetry service if needed
+    // TODO: SNOW-854272 Support telemetry service when using OAuth authentication
     this.telemetryService =
-        ENABLE_TELEMETRY_TO_SF
+        ENABLE_TELEMETRY_TO_SF && credential instanceof KeyPair
             ? new TelemetryService(
                 httpClient, clientName, schemeName + "://" + hostName + ":" + portNum)
             : null;
 
-    // create our security/token manager
-    securityManager = new JWTManager(accountName, userName, keyPair, telemetryService);
-
-    // stash references to the account and user name as well
+    // stash references to the account and username as well
     String account = accountName.toUpperCase();
     String user = userName.toUpperCase();
 
@@ -270,6 +282,27 @@ public RequestBuilder(
     this.host = hostName;
     this.userAgentSuffix = userAgentSuffix;
 
+    // create our security/token manager
+    if (securityManager == null) {
+      if (credential instanceof KeyPair) {
+        this.securityManager =
+            new JWTManager(accountName, userName, (KeyPair) credential, telemetryService);
+      } else if (credential instanceof OAuthCredential) {
+        this.securityManager =
+            new OAuthManager(
+                accountName,
+                userName,
+                (OAuthCredential) credential,
+                makeBaseURI(),
+                telemetryService);
+      } else {
+        throw new IllegalArgumentException(
+            "Credential should be instance of either KeyPair or OAuthCredential");
+      }
+    } else {
+      this.securityManager = securityManager;
+    }
+
     LOGGER.info(
         "Creating a RequestBuilder with arguments : "
             + "Account : {}, User : {}, Scheme : {}, Host : {}, Port : {}, userAgentSuffix: {}",
@@ -351,6 +384,17 @@ private URIBuilder makeBaseURI(UUID requestId) {
       throw new IllegalArgumentException();
     }
 
+    // construct the builder object without param
+    URIBuilder builder = makeBaseURI();
+
+    // set the request id
+    builder.setParameter(REQUEST_ID, requestId.toString());
+
+    return builder;
+  }
+
+  /** Construct a URIBuilder for common parts of request without any parameter */
+  private URIBuilder makeBaseURI() {
     // construct the builder object
     URIBuilder builder = new URIBuilder();
 
@@ -363,9 +407,6 @@ private URIBuilder makeBaseURI(UUID requestId) {
     // set the port name
     builder.setPort(port);
 
-    // set the request id
-    builder.setParameter(REQUEST_ID, requestId.toString());
-
     return builder;
   }
 
@@ -521,17 +562,24 @@ private static void addUserAgent(HttpUriRequest request, String userAgentSuffix)
   }
 
   /**
-   * addToken - adds a the JWT token to a request
+   * addToken - adds a token to a request
    *
    * @param request the URI request
    * @param token the token to add
    */
-  private static void addToken(HttpUriRequest request, String token) {
+  private void addToken(HttpUriRequest request, String token) {
     request.setHeader(HttpHeaders.AUTHORIZATION, BEARER_PARAMETER + token);
-    request.setHeader(SF_HEADER_AUTHORIZATION_TOKEN_TYPE, JWT_TOKEN_TYPE);
+    request.setHeader(SF_HEADER_AUTHORIZATION_TOKEN_TYPE, this.securityManager.getTokenType());
   }
 
-  private static void addHeaders(HttpUriRequest request, String token, String userAgentSuffix) {
+  /**
+   * addHeader - adds necessary header to a request
+   *
+   * @param request the URI request
+   * @param token authentication token
+   * @param userAgentSuffix user agent suffix
+   */
+  private void addHeaders(HttpUriRequest request, String token, String userAgentSuffix) {
     addUserAgent(request, userAgentSuffix);
 
     // Add the auth token
@@ -674,6 +722,23 @@ public HttpPost generateStreamingIngestPostRequest(
     return this.generateStreamingIngestPostRequest(payloadInString, endPoint, message);
   }
 
+  /**
+   * Set refresh token, this method is for refresh token renewal without requiring to restart
+   * client. This method only works when the authorization type is OAuth
+   *
+   * @param refreshToken the new refresh token
+   */
+  public void setRefreshToken(String refreshToken) {
+    if (securityManager instanceof OAuthManager) {
+      ((OAuthManager) securityManager).setRefreshToken(refreshToken);
+    }
+  }
+
+  /** Get authorization type */
+  public String getAuthType() {
+    return securityManager.getTokenType();
+  }
+
   /**
    * Closes the resources being used by RequestBuilder object. {@link SecurityManager} is one such
    * resource which uses a threadpool which needs to be shutdown once SimpleIngestManager is done
@@ -683,7 +748,9 @@ public HttpPost generateStreamingIngestPostRequest(
    */
   public void closeResources() {
     securityManager.close();
-    telemetryService.close();
+    if (telemetryService != null) {
+      telemetryService.close();
+    }
   }
 
   /** Get the telemetry service */
diff --git a/src/main/java/net/snowflake/ingest/connection/SnowflakeOAuthClient.java b/src/main/java/net/snowflake/ingest/connection/SnowflakeOAuthClient.java
new file mode 100644
index 000000000..a90540659
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/connection/SnowflakeOAuthClient.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.connection;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest;
+import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder;
+import net.snowflake.client.jdbc.internal.apache.http.entity.ContentType;
+import net.snowflake.client.jdbc.internal.apache.http.entity.StringEntity;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
+import net.snowflake.client.jdbc.internal.apache.http.util.EntityUtils;
+import net.snowflake.client.jdbc.internal.google.api.client.http.HttpStatusCodes;
+import net.snowflake.client.jdbc.internal.google.gson.JsonObject;
+import net.snowflake.client.jdbc.internal.google.gson.JsonParser;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.HttpUtil;
+import net.snowflake.ingest.utils.SFException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/*
+ * Implementation of Snowflake OAuth Client, used for refreshing an OAuth access token.
+ */
+public class SnowflakeOAuthClient implements OAuthClient {
+
+  static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeOAuthClient.class);
+  private static final String TOKEN_REQUEST_ENDPOINT = "/oauth/token-request";
+
+  // Content type header to specify the encoding
+  private static final String OAUTH_CONTENT_TYPE_HEADER = "application/x-www-form-urlencoded";
+  private static final String GRANT_TYPE_PARAM = "grant_type";
+  private static final String ACCESS_TOKEN = "access_token";
+  private static final String REFRESH_TOKEN = "refresh_token";
+  private static final String EXPIRES_IN = "expires_in";
+
+  // OAuth credential
+  private final AtomicReference<OAuthCredential> oAuthCredential;
+
+  // exact uri for token request
+  private final URI tokenRequestURI;
+
+  // Http client for submitting token refresh request
+  private final CloseableHttpClient httpClient;
+
+  /**
+   * Creates an SnowflakeOAuthClient given account, credential and base uri
+   *
+   * @param accountName - the snowflake account name of this user
+   * @param oAuthCredential - the OAuth credential we're using to connect
+   * @param baseURIBuilder - the uri builder with common scheme, host and port
+   */
+  SnowflakeOAuthClient(
+      String accountName, OAuthCredential oAuthCredential, URIBuilder baseURIBuilder) {
+    this.oAuthCredential = new AtomicReference<>(oAuthCredential);
+
+    // build token request uri
+    baseURIBuilder.setPath(TOKEN_REQUEST_ENDPOINT);
+    try {
+      this.tokenRequestURI = baseURIBuilder.build();
+    } catch (URISyntaxException e) {
+      throw new SFException(e, ErrorCode.MAKE_URI_FAILURE, e.getMessage());
+    }
+
+    this.httpClient = HttpUtil.getHttpClient(accountName);
+  }
+
+  /** Get access token */
+  @Override
+  public AtomicReference<OAuthCredential> getoAuthCredentialRef() {
+    return oAuthCredential;
+  }
+
+  /** Refresh access token using a valid refresh token */
+  @Override
+  public void refreshToken() {
+    String respBodyString = null;
+    try (CloseableHttpResponse httpResponse = httpClient.execute(makeRefreshTokenRequest())) {
+      respBodyString = EntityUtils.toString(httpResponse.getEntity());
+
+      if (httpResponse.getStatusLine().getStatusCode() == HttpStatusCodes.STATUS_CODE_OK) {
+        JsonObject respBody = JsonParser.parseString(respBodyString).getAsJsonObject();
+
+        if (respBody.has(ACCESS_TOKEN) && respBody.has(EXPIRES_IN)) {
+          // Trim surrounding quotation marks
+          String newAccessToken = respBody.get(ACCESS_TOKEN).toString().replaceAll("^\"|\"$", "");
+
+          oAuthCredential.get().setAccessToken(newAccessToken);
+          oAuthCredential.get().setExpiresIn(respBody.get(EXPIRES_IN).getAsInt());
+          return;
+        }
+      }
+      throw new SFException(
+          ErrorCode.OAUTH_REFRESH_TOKEN_ERROR,
+          "Refresh access token fail with response: " + respBodyString);
+    } catch (Exception e) {
+      throw new SFException(ErrorCode.OAUTH_REFRESH_TOKEN_ERROR, e.getMessage());
+    }
+  }
+
+  /** Helper method for making refresh request */
+  private HttpUriRequest makeRefreshTokenRequest() {
+    HttpPost post = new HttpPost(tokenRequestURI);
+    post.addHeader(HttpHeaders.CONTENT_TYPE, OAUTH_CONTENT_TYPE_HEADER);
+    post.addHeader(HttpHeaders.AUTHORIZATION, oAuthCredential.get().getAuthHeader());
+
+    Map<Object, Object> payload = new HashMap<>();
+    try {
+      payload.put(GRANT_TYPE_PARAM, URLEncoder.encode(REFRESH_TOKEN, "UTF-8"));
+      payload.put(
+          REFRESH_TOKEN, URLEncoder.encode(oAuthCredential.get().getRefreshToken(), "UTF-8"));
+    } catch (UnsupportedEncodingException e) {
+      throw new SFException(e, ErrorCode.OAUTH_REFRESH_TOKEN_ERROR, e.getMessage());
+    }
+
+    String payloadString =
+        payload.entrySet().stream()
+            .map(e -> e.getKey() + "=" + e.getValue())
+            .collect(Collectors.joining("&"));
+
+    final StringEntity entity =
+        new StringEntity(payloadString, ContentType.APPLICATION_FORM_URLENCODED);
+    post.setEntity(entity);
+
+    return post;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
index d5f9e5c2e..6921bcabc 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
@@ -29,6 +29,14 @@ public interface SnowflakeStreamingIngestClient extends AutoCloseable {
    */
   String getName();
 
+  /**
+   * Set refresh token, this method is for refresh token renewal without requiring to restart
+   * client. This method only works when the authorization type is OAuth.
+   *
+   * @param refreshToken the new refresh token
+   */
+  void setRefreshToken(String refreshToken);
+
   /**
    * Check whether the client is closed or not, if you want to make sure all data are committed
    * before closing, please call {@link SnowflakeStreamingIngestClient#close()} before closing the
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 3d478624b..7836e7b44 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -38,7 +38,6 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
-import java.security.KeyPair;
 import java.security.NoSuchAlgorithmException;
 import java.security.PrivateKey;
 import java.security.spec.InvalidKeySpecException;
@@ -61,6 +60,7 @@
 import net.snowflake.client.core.SFSessionProperty;
 import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.ingest.connection.IngestResponseException;
+import net.snowflake.ingest.connection.OAuthCredential;
 import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.connection.TelemetryService;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
@@ -170,20 +170,34 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
     if (!isTestMode) {
       // Setup request builder for communication with the server side
       this.role = prop.getProperty(Constants.ROLE);
-      try {
-        KeyPair keyPair =
-            Utils.createKeyPairFromPrivateKey(
-                (PrivateKey) prop.get(SFSessionProperty.PRIVATE_KEY.getPropertyKey()));
-        this.requestBuilder =
-            new RequestBuilder(
-                accountURL,
-                prop.get(USER).toString(),
-                keyPair,
-                this.httpClient,
-                String.format("%s_%s", this.name, System.currentTimeMillis()));
-      } catch (NoSuchAlgorithmException | InvalidKeySpecException e) {
-        throw new SFException(e, ErrorCode.KEYPAIR_CREATION_FAILURE);
+
+      Object credential = null;
+
+      // Authorization type will be set to jwt by default
+      if (prop.getProperty(Constants.AUTHORIZATION_TYPE).equals(Constants.JWT)) {
+        try {
+          credential =
+              Utils.createKeyPairFromPrivateKey(
+                  (PrivateKey) prop.get(SFSessionProperty.PRIVATE_KEY.getPropertyKey()));
+        } catch (NoSuchAlgorithmException | InvalidKeySpecException e) {
+          throw new SFException(e, ErrorCode.KEYPAIR_CREATION_FAILURE);
+        }
+      } else {
+        credential =
+            new OAuthCredential(
+                prop.getProperty(Constants.OAUTH_CLIENT_ID),
+                prop.getProperty(Constants.OAUTH_CLIENT_SECRET),
+                prop.getProperty(Constants.OAUTH_REFRESH_TOKEN));
       }
+      this.requestBuilder =
+          new RequestBuilder(
+              accountURL,
+              prop.get(USER).toString(),
+              credential,
+              this.httpClient,
+              String.format("%s_%s", this.name, System.currentTimeMillis()));
+
+      logger.logInfo("Using {} for authorization", this.requestBuilder.getAuthType());
 
       // Setup client telemetries if needed
       this.setupMetricsForClient();
@@ -775,6 +789,19 @@ ParameterProvider getParameterProvider() {
     return parameterProvider;
   }
 
+  /**
+   * Set refresh token, this method is for refresh token renewal without requiring to restart
+   * client. This method only works when the authorization type is OAuth
+   *
+   * @param refreshToken the new refresh token
+   */
+  @Override
+  public void setRefreshToken(String refreshToken) {
+    if (requestBuilder != null) {
+      requestBuilder.setRefreshToken(refreshToken);
+    }
+  }
+
   /**
    * Registers the performance metrics along with JVM memory and Threads.
    *
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index 41465304a..09814235b 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -22,8 +22,14 @@ public class Constants {
   public static final String SCHEME = "scheme";
   public static final String ACCOUNT_URL = "url";
   public static final String ROLE = "role";
+  public static final String AUTHORIZATION_TYPE = "authorization_type";
+  public static final String JWT = "JWT";
+  public static final String OAUTH = "OAuth";
   public static final String PRIVATE_KEY = "private_key";
   public static final String PRIVATE_KEY_PASSPHRASE = "private_key_passphrase";
+  public static final String OAUTH_CLIENT_ID = "oauth_client_id";
+  public static final String OAUTH_CLIENT_SECRET = "oauth_client_secret";
+  public static final String OAUTH_REFRESH_TOKEN = "oauth_refresh_token";
   public static final String PRIMARY_FILE_ID_KEY =
       "primaryFileId"; // Don't change, should match Parquet Scanner
   public static final long RESPONSE_SUCCESS = 0L; // Don't change, should match server side
@@ -50,6 +56,7 @@ public class Constants {
   public static final int MAX_STREAMING_INGEST_API_CHANNEL_RETRY = 3;
   public static final int STREAMING_INGEST_TELEMETRY_UPLOAD_INTERVAL_IN_SEC = 10;
   public static final long EP_NDV_UNKNOWN = -1L;
+  public static final int MAX_OAUTH_REFRESH_TOKEN_RETRY = 3;
 
   // Channel level constants
   public static final String CHANNEL_STATUS_ENDPOINT = "/v1/streaming/channels/status/";
diff --git a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
index 92968fb80..cae9273f4 100644
--- a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
+++ b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
@@ -36,7 +36,10 @@ public enum ErrorCode {
   CHANNEL_STATUS_INVALID("0028"),
   UNSUPPORTED_DATA_TYPE("0029"),
   INVALID_VALUE_ROW("0030"),
-  MAX_ROW_SIZE_EXCEEDED("0031");
+  MAX_ROW_SIZE_EXCEEDED("0031"),
+  MAKE_URI_FAILURE("0032"),
+  OAUTH_REFRESH_TOKEN_ERROR("0033"),
+  INVALID_CONFIG_PARAMETER("0034");
 
   public static final String errorMessageResource = "net.snowflake.ingest.ingest_error_messages";
 
diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java
index 662063a90..a16a8e3ce 100644
--- a/src/main/java/net/snowflake/ingest/utils/Utils.java
+++ b/src/main/java/net/snowflake/ingest/utils/Utils.java
@@ -103,8 +103,30 @@ public static Properties createProperties(Properties inputProp) {
       properties.put(SFSessionProperty.PRIVATE_KEY.getPropertyKey(), parsePrivateKey(privateKey));
     }
 
-    if (!properties.containsKey(SFSessionProperty.PRIVATE_KEY.getPropertyKey())) {
-      throw new SFException(ErrorCode.MISSING_CONFIG, "private_key");
+    // Use JWT if authorization type not specified
+    if (!properties.containsKey(Constants.AUTHORIZATION_TYPE)) {
+      properties.put(Constants.AUTHORIZATION_TYPE, Constants.JWT);
+    }
+
+    String authType = properties.get(Constants.AUTHORIZATION_TYPE).toString();
+    if (authType.equals(Constants.JWT)) {
+      if (!properties.containsKey(SFSessionProperty.PRIVATE_KEY.getPropertyKey())) {
+        throw new SFException(ErrorCode.MISSING_CONFIG, Constants.PRIVATE_KEY);
+      }
+    } else if (authType.equals(Constants.OAUTH)) {
+      if (!properties.containsKey(Constants.OAUTH_CLIENT_ID)) {
+        throw new SFException(ErrorCode.MISSING_CONFIG, Constants.OAUTH_CLIENT_ID);
+      }
+      if (!properties.containsKey(Constants.OAUTH_CLIENT_SECRET)) {
+        throw new SFException(ErrorCode.MISSING_CONFIG, Constants.OAUTH_CLIENT_SECRET);
+      }
+      if (!properties.containsKey(Constants.OAUTH_REFRESH_TOKEN)) {
+        throw new SFException(ErrorCode.MISSING_CONFIG, Constants.OAUTH_REFRESH_TOKEN);
+      }
+    } else {
+      throw new SFException(
+          ErrorCode.INVALID_CONFIG_PARAMETER,
+          String.format("authorization_type, should be %s or %s", Constants.JWT, Constants.OAUTH));
     }
 
     if (!properties.containsKey(USER)) {
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index e0d7a3e07..d779b5a0c 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -17,6 +17,8 @@
 
 import java.io.IOException;
 import java.math.BigDecimal;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -36,6 +38,7 @@
 import java.util.Properties;
 import java.util.Random;
 import java.util.function.Supplier;
+import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
 import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
@@ -50,6 +53,8 @@ public class TestUtils {
   // profile path, follow readme for the format
   private static final String PROFILE_PATH = "profile.json";
 
+  private static final String OAUTH_INTEGRATION = "OAUTH_INTEGRATION";
+
   private static final ObjectMapper mapper = new ObjectMapper();
 
   private static ObjectNode profile = null;
@@ -153,6 +158,13 @@ public static String getUser() throws Exception {
     return user;
   }
 
+  public static String getAccount() throws Exception {
+    if (profile == null) {
+      init();
+    }
+    return account;
+  }
+
   public static String getAccountURL() throws Exception {
     if (profile == null) {
       init();
@@ -452,6 +464,21 @@ public static Map<String, Object> getRandomRow(Random r, boolean nullable) {
     return row;
   }
 
+  public static URIBuilder getBaseURIBuilder() {
+    return new URIBuilder().setScheme(scheme).setHost(host).setPort(port);
+  }
+
+  public static URI getTokenRequestURI() {
+    URI tokenRequestURI = null;
+    try {
+      tokenRequestURI = getBaseURIBuilder().setPath("/oauth/token-request").build();
+    } catch (URISyntaxException e) {
+      throw new RuntimeException("Fail to construct token request uri", e);
+    }
+
+    return tokenRequestURI;
+  }
+
   private static <T> T nullOrIfNullable(boolean nullable, Random r, Supplier<T> value) {
     return !nullable ? value.get() : (r.nextBoolean() ? value.get() : null);
   }
diff --git a/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java b/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java
new file mode 100644
index 000000000..f1c3b2409
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java
@@ -0,0 +1,39 @@
+package net.snowflake.ingest.connection;
+
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicReference;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.SFException;
+
+/** Mock implementation of {@link OAuthClient}, only use for test */
+public class MockOAuthClient implements OAuthClient {
+  private final AtomicReference<OAuthCredential> oAuthCredential;
+
+  private int futureRefreshFailCount = 0;
+
+  public MockOAuthClient() {
+    OAuthCredential mockOAuthCredential =
+        new OAuthCredential("CLIENT_ID", "CLIENT_SECRET", "REFRESH_TOKEN");
+    oAuthCredential = new AtomicReference<>(mockOAuthCredential);
+    oAuthCredential.get().setExpiresIn(600);
+  }
+
+  @Override
+  public AtomicReference<OAuthCredential> getoAuthCredentialRef() {
+    return oAuthCredential;
+  }
+
+  @Override
+  public void refreshToken() {
+    if (futureRefreshFailCount == 0) {
+      oAuthCredential.get().setAccessToken(UUID.randomUUID().toString());
+      return;
+    }
+    futureRefreshFailCount--;
+    throw new SFException(ErrorCode.OAUTH_REFRESH_TOKEN_ERROR);
+  }
+
+  public void setFutureRefreshFailCount(int futureRefreshFailCount) {
+    this.futureRefreshFailCount = futureRefreshFailCount;
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java b/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java
index 94d83f8eb..51d1d6490 100644
--- a/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java
+++ b/src/test/java/net/snowflake/ingest/connection/SecurityManagerTest.java
@@ -1,13 +1,13 @@
 package net.snowflake.ingest.connection;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
 
 import java.security.KeyPair;
 import java.security.KeyPairGenerator;
-import java.security.NoSuchAlgorithmException;
 import java.util.concurrent.TimeUnit;
+import net.snowflake.ingest.TestUtils;
+import net.snowflake.ingest.utils.Constants;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -18,7 +18,7 @@ public class SecurityManagerTest {
   private KeyPair keyPair;
 
   @Before
-  public void setup() throws NoSuchAlgorithmException {
+  public void setup() throws Exception {
     // first we need to create a keypair
     KeyPairGenerator keyGen = KeyPairGenerator.getInstance(ALGORITHM);
     // we need a 2048 bit RSA key for this test
@@ -27,33 +27,33 @@ public void setup() throws NoSuchAlgorithmException {
     keyPair = keyGen.generateKeyPair();
   }
 
-  /** Test instantiate security manager with invalid params */
-  @Test
-  public void testSecurityManagerInstantiate() {
-    // Try to create jwt manager without required args
-    try {
-      SecurityManager jwtManager = new JWTManager(null, null, null, 3, TimeUnit.SECONDS, null);
-    } catch (IllegalArgumentException e1) {
-      try {
-        SecurityManager jwtManager =
-            new JWTManager("account", "user", null, 3, TimeUnit.SECONDS, null);
-      } catch (IllegalArgumentException e2) {
-        return;
-      }
-    }
-    assertFalse("testSecurityManagerInstantiate failed", true);
+  /** Test instantiate jwt manager with invalid params */
+  @Test(expected = IllegalArgumentException.class)
+  public void testJWTManagerInstantiate() {
+    SecurityManager jwtManager = new JWTManager(null, null, null, 3, TimeUnit.SECONDS, null);
+  }
+
+  /** Test instantiate OAuth manager with invalid params */
+  @Test(expected = IllegalArgumentException.class)
+  public void testOAuthManagerInstantiate() {
+    SecurityManager oAuthManager = new OAuthManager("account", "user", null, null, null);
   }
 
   /** Test get token type, should exactly match the request */
   @Test
-  public void testGetTokenType() {
-    SecurityManager manager = new JWTManager("account", "user", keyPair, 3, TimeUnit.SECONDS, null);
-    assertEquals(manager.getTokenType(), "KEYPAIR_JWT");
+  public void testGetTokenType() throws Exception {
+    SecurityManager jwtManager =
+        new JWTManager("account", "user", keyPair, 3, TimeUnit.SECONDS, null);
+    assertEquals(jwtManager.getTokenType(), "KEYPAIR_JWT");
+
+    SecurityManager oAuthManager =
+        new OAuthManager(TestUtils.getAccount(), TestUtils.getUser(), new MockOAuthClient(), 0.8);
+    assertEquals(oAuthManager.getTokenType(), "OAUTH");
   }
 
   /** Evaluates whether or not we are actually renewing jwt tokens */
   @Test
-  public void doesRegenerateJWTToken() throws InterruptedException {
+  public void testRegenerateJWTToken() throws InterruptedException {
     // create the security manager;
     SecurityManager manager = new JWTManager("account", "user", keyPair, 3, TimeUnit.SECONDS, null);
 
@@ -71,16 +71,68 @@ public void doesRegenerateJWTToken() throws InterruptedException {
     assertNotEquals("The renewal thread should have reset the token", token, manager.getToken());
   }
 
-  /** Test behavior of getting token after refresh failed */
+  /** Check if OAuth token do refresh */
   @Test
-  public void testGetTokenFail() {
+  public void testRefreshOAuthToken() throws Exception {
+
+    // Set update threshold ratio to 5e-3, access token should be refreshed after 3 seconds
+    SecurityManager securityManager =
+        new OAuthManager(TestUtils.getAccount(), TestUtils.getUser(), new MockOAuthClient(), 5e-3);
+
+    String token = securityManager.getToken();
+
+    // if we immediately request the same token, it should be exactly the same
+    assertEquals(
+        "Two token requests prior to renewals should be the same",
+        token,
+        securityManager.getToken());
+
+    // sleep for enough time for the thread renewal thread to run
+    Thread.sleep(6000);
+
+    // ascertain that we have overwritten the token
+    assertNotEquals(
+        "The renewal thread should have reset the token", token, securityManager.getToken());
+  }
+
+  /** Test behavior of getting token after refresh failed */
+  @Test(expected = SecurityException.class)
+  public void testGetJWTTokenFail() {
     SecurityManager manager = new JWTManager("account", "user", keyPair, 3, TimeUnit.SECONDS, null);
     manager.setRefreshFailed(true);
-    try {
-      String token = manager.getToken();
-    } catch (SecurityException e) {
-      return;
-    }
-    assertFalse("testGetTokenFail failed", true);
+    String token = manager.getToken();
+  }
+
+  /** Test behavior of getting token after refresh failed */
+  @Test(expected = SecurityException.class)
+  public void testGetOAuthTokenFail() throws Exception {
+    MockOAuthClient mockOAuthClient = new MockOAuthClient();
+
+    SecurityManager manager =
+        new OAuthManager(TestUtils.getAccount(), TestUtils.getUser(), mockOAuthClient, 0.8);
+    manager.setRefreshFailed(true);
+    String token = manager.getToken();
+  }
+
+  /** Test refresh oauth token fail */
+  @Test(expected = SecurityException.class)
+  public void testOAuthRefreshFail() throws Exception {
+    MockOAuthClient mockOAuthClient = new MockOAuthClient();
+    mockOAuthClient.setFutureRefreshFailCount(Constants.MAX_OAUTH_REFRESH_TOKEN_RETRY);
+
+    SecurityManager securityManager =
+        new OAuthManager(TestUtils.getAccount(), TestUtils.getUser(), mockOAuthClient, 0.8);
+    securityManager.close();
+  }
+
+  /** Test retry, should success */
+  @Test
+  public void testOAuthRefreshRetry() throws Exception {
+    MockOAuthClient mockOAuthClient = new MockOAuthClient();
+    mockOAuthClient.setFutureRefreshFailCount(Constants.MAX_OAUTH_REFRESH_TOKEN_RETRY - 1);
+
+    SecurityManager securityManager =
+        new OAuthManager(TestUtils.getAccount(), TestUtils.getUser(), mockOAuthClient, 0.8);
+    securityManager.close();
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/OAuthBasicTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/OAuthBasicTest.java
new file mode 100644
index 000000000..88480931b
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/OAuthBasicTest.java
@@ -0,0 +1,121 @@
+package net.snowflake.ingest.streaming.internal;
+
+import java.util.Properties;
+import java.util.UUID;
+import net.snowflake.ingest.TestUtils;
+import net.snowflake.ingest.connection.MockOAuthClient;
+import net.snowflake.ingest.connection.OAuthManager;
+import net.snowflake.ingest.connection.RequestBuilder;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ErrorCode;
+import net.snowflake.ingest.utils.SFException;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This test only contains basic construction of client using OAuth authentication. Further
+ * integration test would be added in dew.
+ */
+public class OAuthBasicTest {
+
+  /** Create client with invalid authorization type, this should fail. */
+  @Test
+  public void invalidAuthType() throws Exception {
+    Properties props = TestUtils.getProperties(Constants.BdecVersion.THREE);
+    props.put(Constants.AUTHORIZATION_TYPE, "INVALID_AUTH_TYPE");
+    SFException e =
+        Assert.assertThrows(
+            SFException.class,
+            () ->
+                SnowflakeStreamingIngestClientFactory.builder("MY_CLIENT")
+                    .setProperties(props)
+                    .build());
+    Assert.assertEquals(e.getVendorCode(), ErrorCode.INVALID_CONFIG_PARAMETER.getMessageCode());
+  }
+
+  /** Create client with missing config, this should fail. */
+  @Test
+  public void missingOAuthParam() throws Exception {
+    Properties props = TestUtils.getProperties(Constants.BdecVersion.THREE);
+    props.put(Constants.AUTHORIZATION_TYPE, Constants.OAUTH);
+
+    // Missing oauth_client_id
+    SFException e =
+        Assert.assertThrows(
+            SFException.class,
+            () ->
+                SnowflakeStreamingIngestClientFactory.builder("MY_CLIENT")
+                    .setProperties(props)
+                    .build());
+    Assert.assertEquals(
+        e.getMessage(),
+        new SFException(ErrorCode.MISSING_CONFIG, Constants.OAUTH_CLIENT_ID).getMessage());
+
+    // Missing oauth_client_secret
+    props.put(Constants.OAUTH_CLIENT_ID, "MOCK_CLIENT_ID");
+    e =
+        Assert.assertThrows(
+            SFException.class,
+            () ->
+                SnowflakeStreamingIngestClientFactory.builder("MY_CLIENT")
+                    .setProperties(props)
+                    .build());
+    Assert.assertEquals(
+        e.getMessage(),
+        new SFException(ErrorCode.MISSING_CONFIG, Constants.OAUTH_CLIENT_SECRET).getMessage());
+
+    // Missing oauth_refresh_token
+    props.put(Constants.OAUTH_CLIENT_SECRET, "MOCK_CLIENT_SECRET");
+    e =
+        Assert.assertThrows(
+            SFException.class,
+            () ->
+                SnowflakeStreamingIngestClientFactory.builder("MY_CLIENT")
+                    .setProperties(props)
+                    .build());
+    Assert.assertEquals(
+        e.getMessage(),
+        new SFException(ErrorCode.MISSING_CONFIG, Constants.OAUTH_REFRESH_TOKEN).getMessage());
+  }
+
+  /** Create client with mock credential, should fail when refreshing token */
+  @Test(expected = SecurityException.class)
+  public void testCreateOAuthClient() throws Exception {
+    Properties props = TestUtils.getProperties(Constants.BdecVersion.THREE);
+    props.remove(Constants.PRIVATE_KEY);
+    props.put(Constants.AUTHORIZATION_TYPE, Constants.OAUTH);
+    props.put(Constants.OAUTH_CLIENT_ID, "MOCK_CLIENT_ID");
+    props.put(Constants.OAUTH_CLIENT_SECRET, "MOCK_CLIENT_SECRET");
+    props.put(Constants.OAUTH_REFRESH_TOKEN, "MOCK_REFRESH_TOKEN");
+    SnowflakeStreamingIngestClient client =
+        SnowflakeStreamingIngestClientFactory.builder("MY_CLIENT").setProperties(props).build();
+  }
+
+  @Test
+  public void testSetRefreshToken() throws Exception {
+    SnowflakeStreamingIngestClientInternal<StubChunkData> client =
+        new SnowflakeStreamingIngestClientInternal<>("TEST_CLIENT");
+    MockOAuthClient mockOAuthClient = new MockOAuthClient();
+
+    OAuthManager oAuthManager =
+        new OAuthManager(TestUtils.getAccount(), TestUtils.getUser(), mockOAuthClient, 0.8);
+    RequestBuilder requestBuilder =
+        new RequestBuilder(
+            "MOCK_ACCOUNTNAME",
+            "MOCK_USERNAME",
+            "MOCK_CREDENTIAL",
+            "https",
+            "MOCK_HOST_NAME",
+            443,
+            null,
+            oAuthManager,
+            null,
+            null);
+    client.injectRequestBuilder(requestBuilder);
+
+    String newToken = UUID.randomUUID().toString();
+    client.setRefreshToken(newToken);
+  }
+}

From 737e814f6d6096d14e39359d803d6cb8e5ddda07 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Fri, 14 Jul 2023 15:57:16 -0700
Subject: [PATCH 267/356] [SNOW-843760] Update pom.xml corresponding to Wiz
 vulnerability scan (#546)

* update pom for vulns

* update pom

* use fasterxml version

* remove hadoop exclusion

* mvn install passes

* passes

* autoformatting

* remove dependency pom

* relocate parquet-hadoop

* dont relocate

* relocate just airlift not parquet-hadoop
---
 pom.xml                     | 14 +++++++++++---
 scripts/process_licenses.py |  1 +
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/pom.xml b/pom.xml
index c3994eb00..846f2ccbb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,7 +45,7 @@
     <commonstext.version>1.10.0</commonstext.version>
     <fasterxml.version>2.14.0</fasterxml.version>
     <guava.version>32.0.1-jre</guava.version>
-    <hadoop.version>3.3.5</hadoop.version>
+    <hadoop.version>3.3.6</hadoop.version>
     <jacoco.skip.instrument>true</jacoco.skip.instrument>
     <jacoco.version>0.8.5</jacoco.version>
     <license.processing.dependencyJarsDir>${project.build.directory}/dependency-jars</license.processing.dependencyJarsDir>
@@ -56,9 +56,9 @@
     <maven.compiler.target>1.8</maven.compiler.target>
     <net.minidev.version>2.4.9</net.minidev.version>
     <netty.version>4.1.94.Final</netty.version>
-    <nimbusds.version>9.9.3</nimbusds.version>
+    <nimbusds.version>9.31</nimbusds.version>
     <objenesis.version>3.1</objenesis.version>
-    <parquet.version>1.12.3</parquet.version>
+    <parquet.version>1.13.1</parquet.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <protobuf.version>3.19.6</protobuf.version>
     <shadeBase>net.snowflake.ingest.internal</shadeBase>
@@ -184,6 +184,10 @@
             <groupId>dnsjava</groupId>
             <artifactId>dnsjava</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>io.dropwizard.metrics</groupId>
+            <artifactId>metrics-core</artifactId>
+          </exclusion>
           <exclusion>
             <groupId>javax.activation</groupId>
             <artifactId>activation</artifactId>
@@ -1015,6 +1019,10 @@
                   <pattern>javax.activation</pattern>
                   <shadedPattern>${shadeBase}.javax.activation</shadedPattern>
                 </relocation>
+                <relocation>
+                  <pattern>io.airlift.compress</pattern>
+                  <shadedPattern>${shadeBase}.io.airlift.compress</shadedPattern>
+                </relocation>
               </relocations>
               <filters>
                 <filter>
diff --git a/scripts/process_licenses.py b/scripts/process_licenses.py
index 69fafefed..a17c34843 100644
--- a/scripts/process_licenses.py
+++ b/scripts/process_licenses.py
@@ -56,6 +56,7 @@
     "org.apache.parquet:parquet-common": APACHE_LICENSE,
     "org.apache.parquet:parquet-format-structures": APACHE_LICENSE,
     "com.github.luben:zstd-jni": BSD_2_CLAUSE_LICENSE,
+    "io.airlift:aircompressor": APACHE_LICENSE,
 }
 
 

From f875bef1fac3eaae853d7696ca84375829f65045 Mon Sep 17 00:00:00 2001
From: Alec Huang <alec.huang@snowflake.com>
Date: Fri, 14 Jul 2023 16:56:48 -0700
Subject: [PATCH 268/356] SNOW-532834 Using default role when
 snowflake.role.name is not specified (#507)

---
 .../example/SnowflakeStreamingIngestExample.java          | 2 +-
 src/main/java/net/snowflake/ingest/utils/Utils.java       | 2 +-
 src/test/java/net/snowflake/ingest/TestUtils.java         | 7 +++++--
 .../ingest/streaming/internal/OAuthBasicTest.java         | 6 +++---
 .../ingest/streaming/internal/OpenManyChannelsIT.java     | 2 +-
 .../streaming/internal/StreamingIngestBigFilesIT.java     | 2 +-
 .../ingest/streaming/internal/StreamingIngestIT.java      | 8 +++-----
 .../internal/datatypes/AbstractDataTypeTest.java          | 2 +-
 8 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
index 2b12aba9f..ceb8aaa1d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
+++ b/src/main/java/net/snowflake/ingest/streaming/example/SnowflakeStreamingIngestExample.java
@@ -26,7 +26,7 @@
 public class SnowflakeStreamingIngestExample {
   // Please follow the example in profile_streaming.json.example to see the required properties, or
   // if you have already set up profile.json with Snowpipe before, all you need is to add the "role"
-  // property.
+  // property. If the "role" is not specified, the default user role will be applied.
   private static String PROFILE_PATH = "profile.json";
   private static final ObjectMapper mapper = new ObjectMapper();
 
diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java
index a16a8e3ce..ca39fed0d 100644
--- a/src/main/java/net/snowflake/ingest/utils/Utils.java
+++ b/src/main/java/net/snowflake/ingest/utils/Utils.java
@@ -153,7 +153,7 @@ public static Properties createProperties(Properties inputProp) {
     }
 
     if (!properties.containsKey(Constants.ROLE)) {
-      throw new SFException(ErrorCode.MISSING_CONFIG, "role");
+      logger.logInfo("Snowflake role is not provided, the default user role will be applied.");
     }
 
     /**
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index d779b5a0c..5b8a7f31e 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -223,7 +223,8 @@ public static String getSchema() throws Exception {
     return schema;
   }
 
-  public static Properties getProperties(Constants.BdecVersion bdecVersion) throws Exception {
+  public static Properties getProperties(Constants.BdecVersion bdecVersion, boolean useDefaultRole)
+      throws Exception {
     if (profile == null) {
       init();
     }
@@ -236,7 +237,9 @@ public static Properties getProperties(Constants.BdecVersion bdecVersion) throws
     props.put(SCHEMA, schema);
     props.put(WAREHOUSE, warehouse);
     props.put(PRIVATE_KEY, privateKeyPem);
-    props.put(ROLE, role);
+    if (!useDefaultRole) {
+      props.put(ROLE, role);
+    }
     props.put(ACCOUNT_URL, getAccountURL());
     props.put(BLOB_FORMAT_VERSION, bdecVersion.toByte());
     return props;
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/OAuthBasicTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/OAuthBasicTest.java
index 88480931b..614f606a9 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/OAuthBasicTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/OAuthBasicTest.java
@@ -23,7 +23,7 @@ public class OAuthBasicTest {
   /** Create client with invalid authorization type, this should fail. */
   @Test
   public void invalidAuthType() throws Exception {
-    Properties props = TestUtils.getProperties(Constants.BdecVersion.THREE);
+    Properties props = TestUtils.getProperties(Constants.BdecVersion.THREE, false);
     props.put(Constants.AUTHORIZATION_TYPE, "INVALID_AUTH_TYPE");
     SFException e =
         Assert.assertThrows(
@@ -38,7 +38,7 @@ public void invalidAuthType() throws Exception {
   /** Create client with missing config, this should fail. */
   @Test
   public void missingOAuthParam() throws Exception {
-    Properties props = TestUtils.getProperties(Constants.BdecVersion.THREE);
+    Properties props = TestUtils.getProperties(Constants.BdecVersion.THREE, false);
     props.put(Constants.AUTHORIZATION_TYPE, Constants.OAUTH);
 
     // Missing oauth_client_id
@@ -83,7 +83,7 @@ public void missingOAuthParam() throws Exception {
   /** Create client with mock credential, should fail when refreshing token */
   @Test(expected = SecurityException.class)
   public void testCreateOAuthClient() throws Exception {
-    Properties props = TestUtils.getProperties(Constants.BdecVersion.THREE);
+    Properties props = TestUtils.getProperties(Constants.BdecVersion.THREE, false);
     props.remove(Constants.PRIVATE_KEY);
     props.put(Constants.AUTHORIZATION_TYPE, Constants.OAUTH);
     props.put(Constants.OAUTH_CLIENT_ID, "MOCK_CLIENT_ID");
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/OpenManyChannelsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/OpenManyChannelsIT.java
index 1934b3f77..b27a76396 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/OpenManyChannelsIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/OpenManyChannelsIT.java
@@ -51,7 +51,7 @@ public void setUp() throws Exception {
             String.format(
                 "create or replace table %s.%s.%s (col int)",
                 databaseName, SCHEMA_NAME, TABLE_NAME));
-    Properties props = TestUtils.getProperties(Constants.BdecVersion.THREE);
+    Properties props = TestUtils.getProperties(Constants.BdecVersion.THREE, false);
     if (props.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       props.setProperty(ROLE, "ACCOUNTADMIN");
     }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
index 4d7e71ac7..0b26bd2b6 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
@@ -47,7 +47,7 @@ public void beforeAll() throws Exception {
         .createStatement()
         .execute(String.format("use warehouse %s", TestUtils.getWarehouse()));
 
-    prop = TestUtils.getProperties(Constants.BdecVersion.THREE);
+    prop = TestUtils.getProperties(Constants.BdecVersion.THREE, false);
     if (prop.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       prop.setProperty(ROLE, "ACCOUNTADMIN");
     }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 2bcfbcdb2..dd7ae7aa8 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -3,7 +3,6 @@
 import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
 import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
 import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.ROLE;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.Is.is;
 
@@ -89,10 +88,9 @@ public void beforeAll() throws Exception {
         .createStatement()
         .execute(String.format("use warehouse %s", TestUtils.getWarehouse()));
 
-    prop = TestUtils.getProperties(Constants.BdecVersion.THREE);
-    if (prop.getProperty(ROLE).equals("DEFAULT_ROLE")) {
-      prop.setProperty(ROLE, "ACCOUNTADMIN");
-    }
+    // Test without role param
+    prop = TestUtils.getProperties(Constants.BdecVersion.THREE, true);
+
     client =
         (SnowflakeStreamingIngestClientInternal<?>)
             SnowflakeStreamingIngestClientFactory.builder("client1").setProperties(prop).build();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index 4ff61e142..f449be2d5 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -66,7 +66,7 @@ public void before() throws Exception {
 
     conn.createStatement().execute(String.format("use warehouse %s;", TestUtils.getWarehouse()));
 
-    Properties props = TestUtils.getProperties(Constants.BdecVersion.THREE);
+    Properties props = TestUtils.getProperties(Constants.BdecVersion.THREE, false);
     if (props.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       props.setProperty(ROLE, "ACCOUNTADMIN");
     }

From 927f37e2c0606626dbafbedccd26a62f41e75448 Mon Sep 17 00:00:00 2001
From: Alec Huang <alec.huang@snowflake.com>
Date: Tue, 18 Jul 2023 10:19:05 -0700
Subject: [PATCH 269/356] SNOW-870386 Add default scheme parameters for test
 utils when profile is not specified (#552)

---
 src/test/java/net/snowflake/ingest/TestUtils.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index 5b8a7f31e..b2ebe1bdd 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -96,6 +96,7 @@ public class TestUtils {
   private static String dummyUser = "user";
   private static int dummyPort = 443;
   private static String dummyHost = "snowflake.qa1.int.snowflakecomputing.com";
+  private static String dummyScheme = "http";
 
   /**
    * load all login info from profile
@@ -134,6 +135,7 @@ private static void init() throws Exception {
       user = dummyUser;
       port = dummyPort;
       host = dummyHost;
+      scheme = dummyScheme;
       KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA");
       kpg.initialize(2048);
       keyPair = kpg.generateKeyPair();

From 833e0c6bf6e254c9bcb13135e42ca3ca016bce3b Mon Sep 17 00:00:00 2001
From: Alec Huang <alec.huang@snowflake.com>
Date: Tue, 18 Jul 2023 13:15:34 -0700
Subject: [PATCH 270/356] SNOW-352846 OAuth Authentication: #3 Add error
 message (#553)

---
 .../net/snowflake/ingest/ingest_error_messages.properties      | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
index 160748106..134c3dac5 100644
--- a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
+++ b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
@@ -34,3 +34,6 @@
 0029=Data type not supported: {0}
 0030=The given row cannot be converted to the internal format due to invalid value: {0}
 0031=The given row exceeds the maximum allowed row size {0}
+0032=URI builder fail to build url: {0}
+0033=OAuth token refresh failure: {0}
+0034=Invalid config parameter: {0}

From df65a01f4de0802d0b76be62e58f3996039d397b Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Thu, 20 Jul 2023 17:49:09 +0200
Subject: [PATCH 271/356] NO-SNOW Retry failed requests more aggressively
 (#550)

---
 .../internal/StreamingIngestUtils.java        | 35 +++++++++++++++++--
 .../net/snowflake/ingest/utils/HttpUtil.java  | 15 ++++++--
 .../internal/StreamingIngestUtilsTest.java    | 17 +++++++++
 .../snowflake/ingest/utils/HttpUtilTest.java  | 10 +++++-
 4 files changed, 72 insertions(+), 5 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
index 88b406a62..ee0a2d5d2 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
@@ -34,9 +34,40 @@ public Long apply(T input) {
 
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
-  static void sleepForRetry(int executionCount) {
+  /**
+   * How many milliseconds of exponential backoff to sleep before retrying the request again:
+   *
+   * <ul>
+   *   <li>0 or 1 failure => no sleep
+   *   <li>2 failures => 1s
+   *   <li>3 failures => 2s
+   *   <li>4 or more failures => 4s
+   * </ul>
+   *
+   * @param executionCount How many unsuccessful attempts have been attempted
+   * @return Sleep time in ms
+   */
+  static long getSleepForRetryMs(int executionCount) {
+    if (executionCount < 0) {
+      throw new IllegalArgumentException(
+          String.format(
+              "executionCount must be a non-negative integer, passed: %d", executionCount));
+    } else if (executionCount < 2) {
+      return 0;
+    } else {
+      final int effectiveExecutionCount = Math.min(executionCount, 4);
+      return (1 << (effectiveExecutionCount - 2)) * 1000L;
+    }
+  }
+
+  public static void sleepForRetry(int executionCount) {
+    long sleepForRetryMs = getSleepForRetryMs(executionCount);
+    if (sleepForRetryMs == 0) {
+      return;
+    }
+
     try {
-      Thread.sleep((1 << (executionCount + 1)) * 1000);
+      Thread.sleep(sleepForRetryMs);
     } catch (InterruptedException e) {
       throw new SFException(ErrorCode.INTERNAL_ERROR, e.getMessage());
     }
diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index fcec12392..1ff65a095 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -40,6 +40,7 @@
 import net.snowflake.client.jdbc.internal.apache.http.pool.PoolStats;
 import net.snowflake.client.jdbc.internal.apache.http.protocol.HttpContext;
 import net.snowflake.client.jdbc.internal.apache.http.ssl.SSLContexts;
+import net.snowflake.ingest.streaming.internal.StreamingIngestUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -57,7 +58,14 @@ public class HttpUtil {
   private static final String FIRST_FAULT_TIMESTAMP = "FIRST_FAULT_TIMESTAMP";
   private static final Duration TOTAL_RETRY_DURATION = Duration.of(120, ChronoUnit.SECONDS);
   private static final Duration RETRY_INTERVAL = Duration.of(3, ChronoUnit.SECONDS);
-  private static final int MAX_RETRIES = 3;
+
+  /**
+   * How many times to retry when an IO exception is thrown. Value here is chosen to match the total
+   * value of {@link HttpUtil.TOTAL_RETRY_DURATION} when exponential backoff of up to 4 seconds per
+   * retry is used.
+   */
+  private static final int MAX_RETRIES = 10;
+
   private static volatile CloseableHttpClient httpClient;
 
   private static PoolingHttpClientConnectionManager connectionManager;
@@ -284,13 +292,16 @@ static HttpRequestRetryHandler getHttpRequestRetryHandler() {
         return false;
       }
       if (exception instanceof NoHttpResponseException
-          || exception instanceof javax.net.ssl.SSLException) {
+          || exception instanceof javax.net.ssl.SSLException
+          || exception instanceof java.net.SocketException
+          || exception instanceof java.net.UnknownHostException) {
         LOGGER.info(
             "Retrying request which caused {} with " + "URI:{}, retryCount:{} and maxRetryCount:{}",
             exception.getClass().getName(),
             requestURI,
             executionCount,
             MAX_RETRIES);
+        StreamingIngestUtils.sleepForRetry(executionCount);
         return true;
       }
       LOGGER.info("No retry for URI:{} with exception {}", requestURI, exception.toString());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
index c0807bb81..c119ddd5c 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
@@ -2,6 +2,7 @@
 
 import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CHANNEL_STATUS;
 import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.getSleepForRetryMs;
 import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
@@ -162,4 +163,20 @@ public void testRetriesRecovery() throws Exception {
 
     Assert.assertEquals("honkSuccess", result.getMessage());
   }
+
+  @Test
+  public void testGetSleepForRetry() {
+    Assert.assertEquals(0, getSleepForRetryMs(0));
+    Assert.assertEquals(0, getSleepForRetryMs(1));
+    Assert.assertEquals(1000, getSleepForRetryMs(2));
+    Assert.assertEquals(2000, getSleepForRetryMs(3));
+    Assert.assertEquals(4000, getSleepForRetryMs(4));
+    Assert.assertEquals(4000, getSleepForRetryMs(5));
+    Assert.assertEquals(4000, getSleepForRetryMs(100000));
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testGetSleepForRetryNegative() {
+    getSleepForRetryMs(-1);
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java b/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java
index 5e21cad99..a905848e6 100644
--- a/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java
@@ -5,6 +5,8 @@
 import static org.mockito.Mockito.doReturn;
 
 import java.io.IOException;
+import java.net.SocketException;
+import java.net.UnknownHostException;
 import javax.net.ssl.SSLException;
 import net.snowflake.client.jdbc.internal.apache.http.HttpRequest;
 import net.snowflake.client.jdbc.internal.apache.http.NoHttpResponseException;
@@ -33,9 +35,15 @@ public void testRequestRetryHandler() {
     assertTrue(
         httpRequestRetryHandler.retryRequest(
             new SSLException("Test exception"), 1, httpContextMock));
+    assertTrue(
+        httpRequestRetryHandler.retryRequest(
+            new SocketException("Test exception"), 1, httpContextMock));
+    assertTrue(
+        httpRequestRetryHandler.retryRequest(
+            new UnknownHostException("Test exception"), 1, httpContextMock));
     assertFalse(
         httpRequestRetryHandler.retryRequest(
-            new SSLException("Test exception"), 4, httpContextMock));
+            new SSLException("Test exception"), 11, httpContextMock));
     assertFalse(httpRequestRetryHandler.retryRequest(new IOException(), 1, httpContextMock));
   }
 }

From cc250af03b1a5669fa6ac55a4653e41442ba5c2a Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Sun, 23 Jul 2023 15:45:50 -0700
Subject: [PATCH 272/356] v2.0.2 version upgrade (#557)

Upgrade version to v2.0.2, this will be used as our first GA release!
---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 846f2ccbb..42d0a9779 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.0.2-SNAPSHOT</version>
+  <version>2.0.2</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 0aecf1a67..2c2d263bb 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.0.2-SNAPSHOT";
+  public static final String DEFAULT_VERSION = "2.0.2";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 690a2747d0f823cf147f9b09a752694287f742fb Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Thu, 8 Jun 2023 13:24:11 +0000
Subject: [PATCH 273/356] @snow SNOW-835618 Snowpipe Streaming: send
 uncompressed chunk length from SDK to GS

---
 .../streaming/internal/BlobBuilder.java       |  5 +++--
 .../streaming/internal/ChunkMetadata.java     | 19 +++++++++++++++++++
 .../ingest/streaming/internal/Flusher.java    |  6 +++---
 .../streaming/internal/ParquetFlusher.java    | 12 ++++++------
 .../streaming/internal/FlushServiceTest.java  |  7 ++++++-
 5 files changed, 37 insertions(+), 12 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
index 9442b2774..b88090e01 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobBuilder.java
@@ -114,6 +114,7 @@ static <T> Blob constructBlobAndMetadata(
                 // The paddedChunkLength is used because it is the actual data size used for
                 // decompression and md5 calculation on server side.
                 .setChunkLength(paddedChunkLength)
+                .setUncompressedChunkLength((int) serializedChunk.chunkEstimatedUncompressedSize)
                 .setChannelList(serializedChunk.channelsMetadataList)
                 .setChunkMD5(md5)
                 .setEncryptionKeyId(firstChannelFlushContext.getEncryptionKeyId())
@@ -132,13 +133,13 @@ static <T> Blob constructBlobAndMetadata(
 
         logger.logInfo(
             "Finish building chunk in blob={}, table={}, rowCount={}, startOffset={},"
-                + " uncompressedSize={}, paddedChunkLength={}, encryptedCompressedSize={},"
+                + " estimatedUncompressedSize={}, paddedChunkLength={}, encryptedCompressedSize={},"
                 + " bdecVersion={}",
             filePath,
             firstChannelFlushContext.getFullyQualifiedTableName(),
             serializedChunk.rowCount,
             startOffset,
-            serializedChunk.chunkUncompressedSize,
+            serializedChunk.chunkEstimatedUncompressedSize,
             paddedChunkLength,
             encryptedCompressedChunkDataSize,
             bdecVersion);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChunkMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChunkMetadata.java
index ca2faaa1d..7b42dbc5e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChunkMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChunkMetadata.java
@@ -15,6 +15,7 @@ class ChunkMetadata {
   private final String tableName;
   private Long chunkStartOffset;
   private final Integer chunkLength;
+  private final Integer uncompressedChunkLength;
   private final List<ChannelMetadata> channels;
   private final String chunkMD5;
   private final EpInfo epInfo;
@@ -33,6 +34,9 @@ static class Builder {
     private String tableName;
     private Long chunkStartOffset;
     private Integer chunkLength; // compressedChunkLength
+
+    private Integer uncompressedChunkLength;
+
     private List<ChannelMetadata> channels;
     private String chunkMD5;
     private EpInfo epInfo;
@@ -62,6 +66,15 @@ Builder setChunkLength(Integer chunkLength) {
       return this;
     }
 
+    /**
+     * Currently we send estimated uncompressed size that is close to the actual parquet data size
+     * and mostly about user data but parquet encoding overhead may be slightly different.
+     */
+    public Builder setUncompressedChunkLength(Integer uncompressedChunkLength) {
+      this.uncompressedChunkLength = uncompressedChunkLength;
+      return this;
+    }
+
     Builder setChannelList(List<ChannelMetadata> channels) {
       this.channels = channels;
       return this;
@@ -110,6 +123,7 @@ private ChunkMetadata(Builder builder) {
     this.tableName = builder.tableName;
     this.chunkStartOffset = builder.chunkStartOffset;
     this.chunkLength = builder.chunkLength;
+    this.uncompressedChunkLength = builder.uncompressedChunkLength;
     this.channels = builder.channels;
     this.chunkMD5 = builder.chunkMD5;
     this.epInfo = builder.epInfo;
@@ -152,6 +166,11 @@ Integer getChunkLength() {
     return chunkLength;
   }
 
+  @JsonProperty("chunk_length_uncompressed")
+  public Integer getUncompressedChunkLength() {
+    return uncompressedChunkLength;
+  }
+
   @JsonProperty("channels")
   List<ChannelMetadata> getChannels() {
     return this.channels;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
index e5a562f01..3bb339975 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/Flusher.java
@@ -33,7 +33,7 @@ class SerializationResult {
     final List<ChannelMetadata> channelsMetadataList;
     final Map<String, RowBufferStats> columnEpStatsMapCombined;
     final long rowCount;
-    final float chunkUncompressedSize;
+    final float chunkEstimatedUncompressedSize;
     final ByteArrayOutputStream chunkData;
     final Pair<Long, Long> chunkMinMaxInsertTimeInMs;
 
@@ -41,13 +41,13 @@ public SerializationResult(
         List<ChannelMetadata> channelsMetadataList,
         Map<String, RowBufferStats> columnEpStatsMapCombined,
         long rowCount,
-        float chunkUncompressedSize,
+        float chunkEstimatedUncompressedSize,
         ByteArrayOutputStream chunkData,
         Pair<Long, Long> chunkMinMaxInsertTimeInMs) {
       this.channelsMetadataList = channelsMetadataList;
       this.columnEpStatsMapCombined = columnEpStatsMapCombined;
       this.rowCount = rowCount;
-      this.chunkUncompressedSize = chunkUncompressedSize;
+      this.chunkEstimatedUncompressedSize = chunkEstimatedUncompressedSize;
       this.chunkData = chunkData;
       this.chunkMinMaxInsertTimeInMs = chunkMinMaxInsertTimeInMs;
     }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
index b0c14b12f..87c30207d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
@@ -53,7 +53,7 @@ private SerializationResult serializeFromParquetWriteBuffers(
       throws IOException {
     List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
     long rowCount = 0L;
-    float chunkUncompressedSize = 0f;
+    float chunkEstimatedUncompressedSize = 0f;
     String firstChannelFullyQualifiedTableName = null;
     Map<String, RowBufferStats> columnEpStatsMapCombined = null;
     BdecParquetWriter mergedChannelWriter = null;
@@ -104,7 +104,7 @@ private SerializationResult serializeFromParquetWriteBuffers(
       }
 
       rowCount += data.getRowCount();
-      chunkUncompressedSize += data.getBufferSize();
+      chunkEstimatedUncompressedSize += data.getBufferSize();
 
       logger.logDebug(
           "Parquet Flusher: Finish building channel={}, rowCount={}, bufferSize={} in blob={}",
@@ -121,7 +121,7 @@ private SerializationResult serializeFromParquetWriteBuffers(
         channelsMetadataList,
         columnEpStatsMapCombined,
         rowCount,
-        chunkUncompressedSize,
+        chunkEstimatedUncompressedSize,
         mergedChunkData,
         chunkMinMaxInsertTimeInMs);
   }
@@ -131,7 +131,7 @@ private SerializationResult serializeFromJavaObjects(
       throws IOException {
     List<ChannelMetadata> channelsMetadataList = new ArrayList<>();
     long rowCount = 0L;
-    float chunkUncompressedSize = 0f;
+    float chunkEstimatedUncompressedSize = 0f;
     String firstChannelFullyQualifiedTableName = null;
     Map<String, RowBufferStats> columnEpStatsMapCombined = null;
     List<List<Object>> rows = null;
@@ -183,7 +183,7 @@ private SerializationResult serializeFromJavaObjects(
       rows.addAll(data.getVectors().rows);
 
       rowCount += data.getRowCount();
-      chunkUncompressedSize += data.getBufferSize();
+      chunkEstimatedUncompressedSize += data.getBufferSize();
 
       logger.logDebug(
           "Parquet Flusher: Finish building channel={}, rowCount={}, bufferSize={} in blob={},"
@@ -206,7 +206,7 @@ private SerializationResult serializeFromJavaObjects(
         channelsMetadataList,
         columnEpStatsMapCombined,
         rowCount,
-        chunkUncompressedSize,
+        chunkEstimatedUncompressedSize,
         mergedData,
         chunkMinMaxInsertTimeInMs);
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 38e6ac8ca..c8de18f77 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -843,6 +843,7 @@ public void testBlobBuilder() throws Exception {
             .setOwningTableFromChannelContext(channel1.getChannelContext())
             .setChunkStartOffset(0L)
             .setChunkLength(dataSize)
+            .setUncompressedChunkLength(dataSize * 2)
             .setChannelList(Collections.singletonList(channelMetadata))
             .setChunkMD5("md5")
             .setEncryptionKeyId(1234L)
@@ -867,7 +868,8 @@ public void testBlobBuilder() throws Exception {
               Arrays.copyOfRange(blob, offset, offset += BLOB_TAG_SIZE_IN_BYTES),
               StandardCharsets.UTF_8));
       Assert.assertEquals(
-          bdecVersion, Arrays.copyOfRange(blob, offset, offset += BLOB_VERSION_SIZE_IN_BYTES)[0]);
+          bdecVersion.toByte(),
+          Arrays.copyOfRange(blob, offset, offset += BLOB_VERSION_SIZE_IN_BYTES)[0]);
       long totalSize =
           ByteBuffer.wrap(Arrays.copyOfRange(blob, offset, offset += BLOB_FILE_SIZE_SIZE_IN_BYTES))
               .getLong();
@@ -892,6 +894,9 @@ public void testBlobBuilder() throws Exception {
       Assert.assertEquals(chunkMetadata.getTableName(), map.get("table"));
       Assert.assertEquals(chunkMetadata.getSchemaName(), map.get("schema"));
       Assert.assertEquals(chunkMetadata.getDBName(), map.get("database"));
+      Assert.assertEquals(chunkMetadata.getChunkLength(), map.get("chunk_length"));
+      Assert.assertEquals(
+          chunkMetadata.getUncompressedChunkLength(), map.get("chunk_length_uncompressed"));
       Assert.assertEquals(
           Long.toString(chunkMetadata.getChunkStartOffset() - offset),
           map.get("chunk_start_offset").toString());

From 855b9ee1cd487fe857e7ab84e707da23d10cbfeb Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Mon, 24 Jul 2023 17:18:52 +0200
Subject: [PATCH 274/356] reduce number of rows in BigFilesIT (#559)

---
 .../ingest/streaming/internal/StreamingIngestBigFilesIT.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
index 0b26bd2b6..7994ff7bf 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
@@ -69,7 +69,7 @@ public void testManyRowsMultipleChannelsToMultipleTable()
 
     int numTables = 2;
     int numChannels = 4; // channels are assigned round-robin to tables.
-    int batchSize = 20000;
+    int batchSize = 10000;
     int numBatches = 10; // number of rows PER CHANNEL is batchSize * numBatches
     boolean isNullable = false;
 

From c8f988d2c460d72e9e90ca9619b9e10f8acd1253 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Tue, 1 Aug 2023 14:34:19 -0700
Subject: [PATCH 275/356] NO-SNOW: Support publishing unshaded snapshot release
 to Nexus repo (#561)

Support publishing unshaded snapshot release to Nexus repo
---
 unshaded_snapshot_deploy.sh | 55 +++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100755 unshaded_snapshot_deploy.sh

diff --git a/unshaded_snapshot_deploy.sh b/unshaded_snapshot_deploy.sh
new file mode 100755
index 000000000..dcd578218
--- /dev/null
+++ b/unshaded_snapshot_deploy.sh
@@ -0,0 +1,55 @@
+#!/bin/bash -e
+
+THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+export GPG_KEY_ID="Snowflake Computing"
+export LDAP_USER="$sonatype_user"
+export LDAP_PWD="$sonatype_password"
+
+if [ -z "$GPG_KEY_PASSPHRASE" ]; then
+  echo "[ERROR] GPG passphrase is not specified for $GPG_KEY_ID!"
+  exit 1
+fi
+
+if [ -z "$GPG_PRIVATE_KEY" ]; then
+  echo "[ERROR] GPG private key file is not specified!"
+  exit 1
+fi
+
+echo "[INFO] Import PGP Key"
+if ! gpg --list-secret-key | grep "$GPG_KEY_ID"; then
+  gpg --allow-secret-key-import --import "$GPG_PRIVATE_KEY"
+fi
+
+# copy the settings.xml template and inject credential information
+UNSHADED_SNAPSHOT_DEPLOY_SETTINGS_XML="$THIS_DIR/mvn_settings_unshaded_snapshot_deploy.xml"
+MVN_REPOSITORY_ID=snapshot
+
+cat > $UNSHADED_SNAPSHOT_DEPLOY_SETTINGS_XML << SETTINGS.XML
+<?xml version="1.0" encoding="UTF-8"?>
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+     xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
+  <servers>
+    <server>
+      <id>$MVN_REPOSITORY_ID</id>
+      <username>$LDAP_USER</username>
+      <password>$LDAP_PWD</password>
+    </server>
+  </servers>
+</settings>
+SETTINGS.XML
+
+MVN_OPTIONS+=(
+  "--settings" "$UNSHADED_SNAPSHOT_DEPLOY_SETTINGS_XML"
+  "--batch-mode"
+)
+
+echo "[Info] Sign unshaded package and deploy to staging area"
+project_version=$($THIS_DIR/scripts/get_project_info_from_pom.py $THIS_DIR/pom.xml version)
+echo "[Info] Project version: $project_version"
+$THIS_DIR/scripts/update_project_version.py pom.xml ${project_version} > generated_public_pom.xml
+
+mvn deploy ${MVN_OPTIONS[@]} -Dnot-shadeDep -Dsnapshot-deploy
+
+rm $UNSHADED_SNAPSHOT_DEPLOY_SETTINGS_XML
\ No newline at end of file

From 927dd49d412afecec1abf8d509747d5bf83e0f77 Mon Sep 17 00:00:00 2001
From: Alec Huang <alec.huang@snowflake.com>
Date: Thu, 3 Aug 2023 15:08:43 -0700
Subject: [PATCH 276/356] Update AWS test profile (#564)

---
 profile.json.gpg | Bin 1563 -> 1580 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

diff --git a/profile.json.gpg b/profile.json.gpg
index 11b20def70b656addea2eafcbf36d70c0c7b041e..12955621bd7fae34c150c08d7e7e5e48ff58cd2d 100644
GIT binary patch
literal 1580
zcmV+{2GjYB4Fm}T2+SLhNw~Ey#r@Li0YSn{lKhO!(@m;iRo+V7#ovwRh<?ls_Y0dk
zMgu?n56ZuwxI?V2WRDQx$=KD^sNS)e^F}LcsrR+UcII*D_mP<9e6T&rXl~}NB!#c*
zA;*|m%p(ubHO&Qhm7phd!<Jxh$=);~b7waQ?wx=ncVq;h@-F=N^*TXy$K`8P0*7;L
z;&!vfc!-EtNGV7MDJSlcFtarnjTyJ=jOEn}J+x%5hXhX9fkg(-j-?q>?`6%-QaN8K
zU5xuB9j4U`pOM2#*5(E_trmSHo7XUzWy5zkiH=5z!S<$eSdMTwTIjJd*E|gv!DHsg
zMeF24&toS)JnfZQ2T857_@B5uaKzY{(fu}doM^As1{y|Q!vnH!;pR`DBSJ>BvG;#G
zS#YuVJn&b5Y7Hay&{t?^+BSc);db*}oT|QgA%;=cgm)>5cSv=-Hx_s(iJ(NK)=mk?
zoT{cp45ZUw6ty56@WcwGTVT{fX2;Fue<)5Kj_;XpVB#{lm5-^^s@0~XLd|n09Lcs-
zu=SP4@6E)2=oEN_%p5gq@7^zUiM1cH`?Xo<RRh9#GtrI8QG%Yu{g-y~>nWGX*6uA4
z1pt~XgRTDtytTERTUqjr>HmE~)Ts79J(#@{+2vDm1_+K`9P79HBhz(ik+`a+?3QEK
zH!Ys#qh8;g(8*CyrJJWB+Xt5=Ues!I<^vX?Mh(pAB5t3NCoK*C;_Nq!In%Zd&ReY%
zm_On8_OLg4K@!%Fk=ep8Mmx?<SDSS_yH<It5bMQ63Q?d!|G*=jjVYSy9O3#mBQJGo
zzh>2&3SwUSUA-$p*e$tNi*QH7$Nu{gZUM3@G&US(EA<2ZrN?5fG6{2*I1Sa#jHz#x
zD^Ec>sH$8xH}lmtd4fSG<5sgC6TPCgM#RX`{!41m{$*4p6!JeJJlTtx!-UyOv{lEe
zmMd2%gRi~<RSv6kGayhy@|w{8;9LKLB;QIn0@le$KR%l6FaKJ(q5OkaA&Viro1{2#
zHJlOt@%J*~e@tGL6>VB0F9+JgMox#qI41x@Ktd6zL!DN^3c_W=-eYKTjIJW}LO`K{
zD@x)J%NgS#Crxux<iwKz)4VaVOmN4CPZP0E{N@nLQ+#j8K?i_SwcP}3lm=?0zYq0^
z*wZ4$!>01R^%%t0y;b9l3^pb`uI*6;oa-TFjcEi?jL8JZcSVtZa2+4iaPUmYE~nFg
zX@wr&wlryb8{PGe(U&KBJ$j`JR+eMY-JVLilF`B~Zq_ahu`2;!Ul55CzrEc%zD@`T
z-S2vJDc(<hJ%po9_c{X0SEm4I6P#M+Z*SST&Nkj$E33JZRH#;G`as$8Zw=IH6kB6&
zNH{C1?|R5X=`@;XVNkfYn!<ANU@MJ{6%nR!dyBh@+u@l+9YM<J`D7YFH7SPq2Vr|@
z0CnKvXQ?&k5<pJUN6%ZVSEKIqd0JSFOxqO9!sqtOtT^U3ag=Zg#=$$3Xs&2=o}j-X
zU}j>QbW~YEP-8nYOd(Yf#^TqNu0AIQNJyB9Gjcft&^#K}IP3J(Om8c%2hy~Y2MN1%
zs2`+oqcScBLJkK)oeiUHf{|UkFh81P#@hT3w@e_`3{Fq?a<~p{8E7q#lO8+b0T=V1
zC+i|-ov|^R4kBb@tSt@F!E&5(N4&T(%+$0DaZW#m`PH5Q9*ORhi{-<CC)Hi&4!R^W
zf#-%0<3?92r*XrnKrsCUA72hUSHCMJlyxJYNaik&+&%uxv=q}^x;H5<+JdSSt7NY`
zS>2{4?blj+UcciIrHgXJ$2C4cY&`;h-KQ80T5x)y^q0c{+e4Ji!_ygE+A2pd#fw39
zLlwzZpxYRQ^_Tw7E7|{yMoZ8_HFU+Tb}q@l#ksGA43B609FfA8k3d2mZ>wmK!C<!g
z1wE3{ry-OWATaSbGHFqYbRe9c0F&vvF|)VEaj@yC3($M@8*Qhr*q+wSocrt=h?jh(
z!x2=!!^^~U`&NZXP9y2`faBgZ8GmJ}yHy00sV-G@L9qq&)!f_ubC_7C>am8gn$5SZ
efEj+kHW5f(W{7KkHnmSiXLM%;jwTgR+#+}3i4r~l

literal 1563
zcmV+$2ITpS4Fm}T0<<W_Uy&EkCFj!W0kgBb&Mift0gFK6_^r>Vr_;fzw)&YAkc^@2
zU`Od>pMWugzROE;$qdia@4d?AwRM4t<Gr{v-n02o-=wJ#1a1)|Hh4#-o&>wUW4z}#
za^5rMESIhDcuBou-8!_d^#m(U`1SpC+ophr`-w5eR}I`R`5`Val&nx<d)=p3&)~Kc
z$iJbdHOo^|(B}!wKA3C0Jg)z^9zb({oY}2jl{$s?Zc#U&h4w&~hyHQAlmg>`bIBo^
zkGS#<ge8#{2BWMVii~cuvQ7i*>4;28a97A-+E&LwvFEUo#Vte`QdD*Nugpe2@tOlA
z#Q#x67CblcQ(VKFAv9^$K*O~^v-4qhU;Slv)#eETznNos)10C~rR6G4H@kO#mSc_O
zN6^9fL|<m*6~4Mm^ys`|eQz$S2h~1csJDb#;QRkU&=u<=hv`S?zNwB48l<;OYq~{J
zbU{q(m>idcn=KPh(8b@I?)G+wk1ZacA?S(bqGhQ&@OybORe~yLdan=(p~u~R#W+#*
z)cwzg-JLLfcZldSPxck5D0acK%e-Pa9d^AzQUzm-Y-ob6RCin<)obuR&%XCIG`DyD
zw{Ne(dJ6#XXh=yI3)q<#iVt;YX4&B!!s)Aa!UZ_4C!Ae$CX0!|==i#xvRZg46M;9L
z&SG9UmEGHA&i2H#Z-10|TPKpqtYC2U-*kZN`Y`7B+<cr4Nl%mBVO1-P?yGR7YqqP`
z@p1){jP^wCL^oq-;aqA?hL}K%tIw$e#KainE?CYjDwju^av-D((52UM;rn5*;f1-O
zz`!7@7;B{9<J!#_aK@DTs(%U)8C#@^A&(IK4O$qYDa>xhTn7L%oogg>DD@dTrcFEV
zQ!cC&`+E4AVB`4EZB(c8_vrg?(l)@Poj+8v|5M}7#U`Xet>8}<OhsF0yEIu*8d^G>
zwkkSbq$Wpt^lVXoiu2P(YFmPjLx`;}@w1spKfPR);*C)H5xV%Q@DfkWR*Kq1!-C`u
zwxyqoQCZb8h;`ytwHlWbW84{As@=E6d(^7n0tMM;w$@Egt9cYm&->ZKHOoV3fyegi
zt8Y;Put(1L0geb!g(-u7dIo)5dROY|0@CPemr$Ztz#3SjEa6v-uO`A2QzZNmT<Rkt
zPxxA*%DkKF77j{#$6FMgs=PM-^;ESM{YBJPu{O_i>0G>qK1;+sQ`^MCfblb$QSGL3
z^Gt66Y$TW#*?@<-!Eut<fr74u4pz-i%9LfDJlZNRS#N$CiOJ*1(wJiI`5iiVz;c)8
zsWDENGZ>(W<d1x;3KepiZ=34@x{ZzW%R`dKRpwqXA~C#%zQu$kncTDN7W2tvy1IS4
zVJ4`E-F85<=~z5$yCbQCzP>!}yjwl0tgBT@7~_j(dFUrYKv1sOA40=&0B(sUWDDtY
zjdWBodN(<vPp2XPjnnd$S0ClWdZm7m`$&e%+j||%asK)87{pV~=C9kl-v>6zP_~X0
zMIb|MNk);SDrILXY2<(a9>uuOVHpyWEOv#;CN%~^^$R49`hvg1!FX+_zX;2sZ?ES=
zSC;(XrG*<6`{}z`z}WP({yK1X+zbh9+gLW*2dP%J#<ht>({Y2fM2H4dzG0oFB(zzz
zrj7WgxmtBPx@=cl7xX?ms`ngAP9@s1!e&%-xHpjc{2S<X{BKM@PAj+Ljdgka;mh<s
z5QI~osYzyqtwRGulQmsSZ3Wnk@wHiHIh8dlJR1`_ans=xur^+vXngKDHyHw98zxB>
zVXL@P$#`iAK@Vv1iq8~WZL$AUMQ_<ZEs8nn(ssRd2^e8oed<}Vp<Z#Xr(Vsuccs@K
z<zklDk)EQAYdk%=91=0IByPYO0c>6BRu?Q$Tc%(Bov2*@j?X2k#%T2DWVff%Y|O!c
z(%zo%(CA<P4H*qJl9=1w$;k1xDGKulr!#u0{C@3$q>L57<!4;WXG>LRhkmZU0BccC
zA6%paaC-~t-#4s#bPJds@b|`^sUVN6Vu|z;PrBjNnXnho^m*|GIvOC78Z(L}6?;Qn
Nw+OlG#sR`Nj*<3oBHsW2


From f79784176146d9841dd37cff98bd1d15c310aef4 Mon Sep 17 00:00:00 2001
From: Dhara Shah <dhara.shah@snowflake.com>
Date: Tue, 8 Aug 2023 09:18:03 -0700
Subject: [PATCH 277/356] changed version from 2.0.2 to 2.0.3-SNAPSHOT (#565)

---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 42d0a9779..92e544855 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.0.2</version>
+  <version>2.0.3-SNAPSHOT</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 2c2d263bb..0debf15d0 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.0.2";
+  public static final String DEFAULT_VERSION = "2.0.3-SNAPSHOT";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 2584e77527fb16d12eb68f7d72bc41e480dcb975 Mon Sep 17 00:00:00 2001
From: Alec Huang <alec.huang@snowflake.com>
Date: Fri, 11 Aug 2023 09:03:31 -0700
Subject: [PATCH 278/356] SNOW-877039 Ingest SDK retry for INVALID JWT Response
 (#562)

---
 .../snowflake/ingest/SimpleIngestManager.java |  17 +--
 .../ingest/connection/JWTManager.java         |  14 +--
 .../ingest/connection/OAuthManager.java       |   3 +-
 .../ingest/connection/RequestBuilder.java     |  23 ++--
 .../ingest/connection/SecurityManager.java    |   3 +
 .../connection/ServiceResponseHandler.java    |  98 +++++++++++++----
 .../internal/StreamingIngestUtils.java        |   9 +-
 .../net/snowflake/ingest/SimpleIngestIT.java  |   9 ++
 .../internal/StreamingIngestUtilsTest.java    | 103 +++++++++++++++++-
 9 files changed, 227 insertions(+), 52 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
index aa8a00bf9..17caa2b9b 100644
--- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
+++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
@@ -531,7 +531,8 @@ public IngestResponse ingestFiles(
     // send the request and get a response....
     try (CloseableHttpResponse response = httpClient.execute(httpPostForIngestFile)) {
       LOGGER.info("Attempting to unmarshall insert response - {}", response);
-      return ServiceResponseHandler.unmarshallIngestResponse(response, requestId);
+      return ServiceResponseHandler.unmarshallIngestResponse(
+          response, requestId, httpClient, httpPostForIngestFile, builder);
     }
   }
 
@@ -561,7 +562,8 @@ public HistoryResponse getHistory(UUID requestId, Integer recentSeconds, String
     // send the request and get a response...
     try (CloseableHttpResponse response = httpClient.execute(httpGetHistory)) {
       LOGGER.info("Attempting to unmarshall history response - {}", response);
-      return ServiceResponseHandler.unmarshallHistoryResponse(response, requestId);
+      return ServiceResponseHandler.unmarshallHistoryResponse(
+          response, requestId, httpClient, httpGetHistory, builder);
     }
   }
 
@@ -588,12 +590,13 @@ public HistoryRangeResponse getHistoryRange(
       requestId = UUID.randomUUID();
     }
 
-    try (CloseableHttpResponse response =
-        httpClient.execute(
-            builder.generateHistoryRangeRequest(
-                requestId, pipe, startTimeInclusive, endTimeExclusive))) {
+    HttpGet request =
+        builder.generateHistoryRangeRequest(requestId, pipe, startTimeInclusive, endTimeExclusive);
+
+    try (CloseableHttpResponse response = httpClient.execute(request)) {
       LOGGER.info("Attempting to unmarshall history range response - {}", response);
-      return ServiceResponseHandler.unmarshallHistoryRangeResponse(response, requestId);
+      return ServiceResponseHandler.unmarshallHistoryRangeResponse(
+          response, requestId, httpClient, request, builder);
     }
   }
 
diff --git a/src/main/java/net/snowflake/ingest/connection/JWTManager.java b/src/main/java/net/snowflake/ingest/connection/JWTManager.java
index 1484a3ce9..691348a34 100644
--- a/src/main/java/net/snowflake/ingest/connection/JWTManager.java
+++ b/src/main/java/net/snowflake/ingest/connection/JWTManager.java
@@ -22,7 +22,7 @@
  *
  * @author obabarinsa
  */
-final class JWTManager extends SecurityManager {
+public final class JWTManager extends SecurityManager {
   // the token lifetime is 59 minutes
   private static final float LIFETIME_IN_MINUTES = 59;
 
@@ -66,11 +66,10 @@ final class JWTManager extends SecurityManager {
     this.keyPair = keyPair;
 
     // generate our first token
-    regenerateToken();
+    refreshToken();
 
     // schedule all future renewals
-    tokenRefresher.scheduleAtFixedRate(
-        this::regenerateToken, timeTillRenewal, timeTillRenewal, unit);
+    tokenRefresher.scheduleAtFixedRate(this::refreshToken, timeTillRenewal, timeTillRenewal, unit);
   }
 
   /**
@@ -82,7 +81,7 @@ final class JWTManager extends SecurityManager {
    * @param keyPair - the public/private key pair we're using to connect
    * @param telemetryService reference to the telemetry service
    */
-  JWTManager(
+  public JWTManager(
       String accountName, String username, KeyPair keyPair, TelemetryService telemetryService) {
     this(
         accountName,
@@ -94,7 +93,7 @@ final class JWTManager extends SecurityManager {
   }
 
   @Override
-  String getToken() {
+  public String getToken() {
     // if we failed to regenerate the token at some point, throw
     if (refreshFailed.get()) {
       LOGGER.error("getToken request failed due to token regeneration failure");
@@ -110,7 +109,8 @@ String getTokenType() {
   }
 
   /** regenerateToken - Regenerates our Token given our current user, account and keypair */
-  private void regenerateToken() {
+  @Override
+  void refreshToken() {
     // create our JWT claim builder object
     JWTClaimsSet.Builder builder = new JWTClaimsSet.Builder();
 
diff --git a/src/main/java/net/snowflake/ingest/connection/OAuthManager.java b/src/main/java/net/snowflake/ingest/connection/OAuthManager.java
index 8ba2c05a0..fefb94299 100644
--- a/src/main/java/net/snowflake/ingest/connection/OAuthManager.java
+++ b/src/main/java/net/snowflake/ingest/connection/OAuthManager.java
@@ -138,7 +138,8 @@ void setRefreshToken(String refreshToken) {
   }
 
   /** refreshToken - Get new access token using refresh_token, client_id, client_secret */
-  private void refreshToken() {
+  @Override
+  void refreshToken() {
     for (int retries = 0; retries < Constants.MAX_OAUTH_REFRESH_TOKEN_RETRY; retries++) {
       try {
         oAuthClient.refreshToken();
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 0debf15d0..ffe2fff13 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -565,10 +565,9 @@ private static void addUserAgent(HttpUriRequest request, String userAgentSuffix)
    * addToken - adds a token to a request
    *
    * @param request the URI request
-   * @param token the token to add
    */
-  private void addToken(HttpUriRequest request, String token) {
-    request.setHeader(HttpHeaders.AUTHORIZATION, BEARER_PARAMETER + token);
+  public void addToken(HttpUriRequest request) {
+    request.setHeader(HttpHeaders.AUTHORIZATION, BEARER_PARAMETER + securityManager.getToken());
     request.setHeader(SF_HEADER_AUTHORIZATION_TOKEN_TYPE, this.securityManager.getTokenType());
   }
 
@@ -576,14 +575,13 @@ private void addToken(HttpUriRequest request, String token) {
    * addHeader - adds necessary header to a request
    *
    * @param request the URI request
-   * @param token authentication token
    * @param userAgentSuffix user agent suffix
    */
-  private void addHeaders(HttpUriRequest request, String token, String userAgentSuffix) {
+  private void addHeaders(HttpUriRequest request, String userAgentSuffix) {
     addUserAgent(request, userAgentSuffix);
 
     // Add the auth token
-    addToken(request, token);
+    addToken(request);
 
     // Add Accept header
     request.setHeader(HttpHeaders.ACCEPT, HTTP_HEADER_CONTENT_TYPE_JSON);
@@ -610,7 +608,7 @@ public HttpPost generateInsertRequest(
     // Make the post request
     HttpPost post = new HttpPost(insertURI);
 
-    addHeaders(post, securityManager.getToken(), this.userAgentSuffix);
+    addHeaders(post, this.userAgentSuffix);
 
     // the entity for the containing the json
     final StringEntity entity =
@@ -638,7 +636,7 @@ public HttpGet generateHistoryRequest(
     // make the get request
     HttpGet get = new HttpGet(historyURI);
 
-    addHeaders(get, securityManager.getToken(), this.userAgentSuffix);
+    addHeaders(get, this.userAgentSuffix);
 
     return get;
   }
@@ -664,7 +662,7 @@ public HttpGet generateHistoryRangeRequest(
 
     HttpGet get = new HttpGet(historyRangeURI);
 
-    addHeaders(get, securityManager.getToken(), this.userAgentSuffix /*User agent information*/);
+    addHeaders(get, this.userAgentSuffix /*User agent information*/);
 
     return get;
   }
@@ -692,7 +690,7 @@ public HttpPost generateStreamingIngestPostRequest(
     // Make the post request
     HttpPost post = new HttpPost(uri);
 
-    addHeaders(post, securityManager.getToken(), this.userAgentSuffix /*User agent information*/);
+    addHeaders(post, this.userAgentSuffix /*User agent information*/);
 
     // The entity for the containing the json
     final StringEntity entity = new StringEntity(payload, ContentType.APPLICATION_JSON);
@@ -739,6 +737,11 @@ public String getAuthType() {
     return securityManager.getTokenType();
   }
 
+  /** Refresh token manually */
+  public void refreshToken() {
+    securityManager.refreshToken();
+  }
+
   /**
    * Closes the resources being used by RequestBuilder object. {@link SecurityManager} is one such
    * resource which uses a threadpool which needs to be shutdown once SimpleIngestManager is done
diff --git a/src/main/java/net/snowflake/ingest/connection/SecurityManager.java b/src/main/java/net/snowflake/ingest/connection/SecurityManager.java
index fd6b3daff..3fe89aeff 100644
--- a/src/main/java/net/snowflake/ingest/connection/SecurityManager.java
+++ b/src/main/java/net/snowflake/ingest/connection/SecurityManager.java
@@ -90,5 +90,8 @@ private String parseAccount(final String accountName) {
   /** getTokenType - returns the token type, either "KEYPAIR_JWT" or "OAUTH" */
   abstract String getTokenType();
 
+  /** refreshToken - regenerate or fetch a new token */
+  abstract void refreshToken();
+
   public abstract void close();
 }
diff --git a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java
index 807db49f3..f6c7a6c1b 100644
--- a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java
+++ b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java
@@ -12,6 +12,10 @@
 import net.snowflake.client.jdbc.internal.apache.http.HttpResponse;
 import net.snowflake.client.jdbc.internal.apache.http.HttpStatus;
 import net.snowflake.client.jdbc.internal.apache.http.StatusLine;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpGet;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.client.jdbc.internal.apache.http.util.EntityUtils;
 import net.snowflake.ingest.utils.BackOffException;
 import org.slf4j.Logger;
@@ -74,12 +78,20 @@ private static boolean isStatusOK(StatusLine statusLine) {
    *
    * @param response the HTTPResponse we want to distill into an IngestResponse
    * @param requestId
+   * @param httpClient HttpClient for retries
+   * @param httpPostForIngestFile HttpRequest for retries
+   * @param builder RequestBuilder for retries
    * @return An IngestResponse with all of the parsed out information
    * @throws IOException if our entity is somehow corrupt or we can't get it
    * @throws IngestResponseException - if we have an uncategorized network issue
    * @throws BackOffException - if we have a 503 issue
    */
-  public static IngestResponse unmarshallIngestResponse(HttpResponse response, UUID requestId)
+  public static IngestResponse unmarshallIngestResponse(
+      HttpResponse response,
+      UUID requestId,
+      CloseableHttpClient httpClient,
+      HttpPost httpPostForIngestFile,
+      RequestBuilder builder)
       throws IOException, IngestResponseException, BackOffException {
     // we can't unmarshall a null response
     if (response == null) {
@@ -88,7 +100,9 @@ public static IngestResponse unmarshallIngestResponse(HttpResponse response, UUI
     }
 
     // handle the exceptional status code
-    handleExceptionalStatus(response, requestId, ApiName.INSERT_FILES);
+    response =
+        handleExceptionalStatus(
+            response, requestId, ApiName.INSERT_FILES, httpClient, httpPostForIngestFile, builder);
 
     String blob = consumeAndReturnResponseEntityAsString(response.getEntity());
 
@@ -102,12 +116,20 @@ public static IngestResponse unmarshallIngestResponse(HttpResponse response, UUI
    *
    * @param response the HttpResponse object we are trying to deserialize
    * @param requestId
+   * @param httpClient HttpClient for retries
+   * @param httpGetHistory HttpRequest for retries
+   * @param builder RequestBuilder for retries
    * @return a HistoryResponse with all the parsed out information
    * @throws IOException if our entity is somehow corrupt or we can't get it
    * @throws IngestResponseException - if we have an uncategorized network issue
    * @throws BackOffException - if we have a 503 issue
    */
-  public static HistoryResponse unmarshallHistoryResponse(HttpResponse response, UUID requestId)
+  public static HistoryResponse unmarshallHistoryResponse(
+      HttpResponse response,
+      UUID requestId,
+      CloseableHttpClient httpClient,
+      HttpGet httpGetHistory,
+      RequestBuilder builder)
       throws IOException, IngestResponseException, BackOffException {
     // we can't unmarshall a null response
     if (response == null) {
@@ -116,7 +138,9 @@ public static HistoryResponse unmarshallHistoryResponse(HttpResponse response, U
     }
 
     // handle the exceptional status code
-    handleExceptionalStatus(response, requestId, ApiName.INSERT_REPORT);
+    response =
+        handleExceptionalStatus(
+            response, requestId, ApiName.INSERT_REPORT, httpClient, httpGetHistory, builder);
 
     String blob = consumeAndReturnResponseEntityAsString(response.getEntity());
 
@@ -129,13 +153,20 @@ public static HistoryResponse unmarshallHistoryResponse(HttpResponse response, U
    *
    * @param response the HttpResponse object we are trying to deserialize
    * @param requestId
+   * @param httpClient HttpClient for retries
+   * @param request HttpRequest for retries
+   * @param builder HttpBuilder for retries
    * @return HistoryRangeResponse
    * @throws IOException if our entity is somehow corrupt or we can't get it
    * @throws IngestResponseException - if we have an uncategorized network issue
    * @throws BackOffException - if we have a 503 issue
    */
   public static HistoryRangeResponse unmarshallHistoryRangeResponse(
-      HttpResponse response, UUID requestId)
+      HttpResponse response,
+      UUID requestId,
+      CloseableHttpClient httpClient,
+      HttpGet request,
+      RequestBuilder builder)
       throws IOException, IngestResponseException, BackOffException {
 
     // we can't unmarshall a null response
@@ -145,7 +176,9 @@ public static HistoryRangeResponse unmarshallHistoryRangeResponse(
     }
 
     // handle the exceptional status code
-    handleExceptionalStatus(response, requestId, ApiName.LOAD_HISTORY_SCAN);
+    response =
+        handleExceptionalStatus(
+            response, requestId, ApiName.LOAD_HISTORY_SCAN, httpClient, request, builder);
 
     String blob = consumeAndReturnResponseEntityAsString(response.getEntity());
     // read out our blob into a pojo
@@ -164,7 +197,12 @@ public static HistoryRangeResponse unmarshallHistoryRangeResponse(
    * @throws IngestResponseException if received an exceptional status code
    */
   public static <T> T unmarshallStreamingIngestResponse(
-      HttpResponse response, Class<T> valueType, ApiName apiName)
+      HttpResponse response,
+      Class<T> valueType,
+      ApiName apiName,
+      CloseableHttpClient httpClient,
+      HttpUriRequest request,
+      RequestBuilder requestBuilder)
       throws IOException, IngestResponseException {
     // We can't unmarshall a null response
     if (response == null) {
@@ -173,7 +211,8 @@ public static <T> T unmarshallStreamingIngestResponse(
     }
 
     // Handle the exceptional status code
-    handleExceptionalStatus(response, null, apiName);
+    response =
+        handleExceptionalStatus(response, null, apiName, httpClient, request, requestBuilder);
 
     // Grab the string version of the response entity
     String blob = consumeAndReturnResponseEntityAsString(response.getEntity());
@@ -187,36 +226,53 @@ public static <T> T unmarshallStreamingIngestResponse(
    *
    * @param response HttpResponse
    * @param requestId
+   * @param apiName enum to represent the corresponding api name
+   * @param httpClient HttpClient for retries
+   * @param request HttpRequest for retries
+   * @param requestBuilder RequestBuilder for retries
+   * @return modified HttpResponse
    * @throws IOException if our entity is somehow corrupt or we can't get it
    * @throws IngestResponseException - for all other non OK status
    * @throws BackOffException - if we have a 503 issue
    */
-  private static void handleExceptionalStatus(
-      HttpResponse response, UUID requestId, ApiName apiName)
+  private static HttpResponse handleExceptionalStatus(
+      HttpResponse response,
+      UUID requestId,
+      ApiName apiName,
+      CloseableHttpClient httpClient,
+      HttpUriRequest request,
+      RequestBuilder requestBuilder)
       throws IOException, IngestResponseException, BackOffException {
-    StatusLine statusLine = response.getStatusLine();
-    if (!isStatusOK(statusLine)) {
+    if (!isStatusOK(response.getStatusLine())) {
+      StatusLine statusLine = response.getStatusLine();
+      LOGGER.warn(
+          "{} Status hit from {}, requestId:{}",
+          statusLine.getStatusCode(),
+          apiName,
+          requestId == null ? "" : requestId.toString());
+
       // if we have a 503 exception throw a backoff
       switch (statusLine.getStatusCode()) {
           // If we have a 503, BACKOFF
         case HttpStatus.SC_SERVICE_UNAVAILABLE:
-          LOGGER.warn(
-              "503 Status hit from {}, backoff, requestId:{}",
-              apiName,
-              requestId == null ? "" : requestId.toString());
           throw new BackOffException();
+        case HttpStatus.SC_UNAUTHORIZED:
+          LOGGER.warn("Authorization failed, refreshing Token succeeded, retry");
+          requestBuilder.refreshToken();
+          requestBuilder.addToken(request);
+          response = httpClient.execute(request);
+          if (!isStatusOK(response.getStatusLine())) {
+            throw new SecurityException("Authorization failed after retry");
+          }
+          break;
         default:
-          LOGGER.error(
-              "Exceptional Status Code from {}: {}, requestId:{}",
-              apiName,
-              statusLine.getStatusCode(),
-              requestId == null ? "" : requestId.toString());
           String blob = consumeAndReturnResponseEntityAsString(response.getEntity());
           throw new IngestResponseException(
               statusLine.getStatusCode(),
               IngestResponseException.IngestExceptionBody.parseBody(blob));
       }
     }
+    return response;
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
index ee0a2d5d2..56e960064 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java
@@ -9,6 +9,7 @@
 import java.util.Map;
 import java.util.function.Function;
 import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse;
+import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest;
 import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.ingest.connection.IngestResponseException;
 import net.snowflake.ingest.connection.RequestBuilder;
@@ -125,13 +126,13 @@ static <T> T executeWithRetries(
       throws IOException, IngestResponseException {
     int retries = 0;
     T response;
+    HttpUriRequest request =
+        requestBuilder.generateStreamingIngestPostRequest(payload, endpoint, message);
     do {
-      try (CloseableHttpResponse httpResponse =
-          httpClient.execute(
-              requestBuilder.generateStreamingIngestPostRequest(payload, endpoint, message))) {
+      try (CloseableHttpResponse httpResponse = httpClient.execute(request)) {
         response =
             ServiceResponseHandler.unmarshallStreamingIngestResponse(
-                httpResponse, targetClass, apiName);
+                httpResponse, targetClass, apiName, httpClient, request, requestBuilder);
       }
 
       if (statusGetter.apply(response) == RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST) {
diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
index f1c4c06f1..b1eafc93e 100644
--- a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
@@ -9,6 +9,7 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.net.URL;
+import java.time.Instant;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Random;
@@ -21,6 +22,7 @@
 import net.snowflake.client.jdbc.internal.apache.http.Header;
 import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders;
 import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost;
+import net.snowflake.ingest.connection.HistoryRangeResponse;
 import net.snowflake.ingest.connection.HistoryResponse;
 import net.snowflake.ingest.connection.IngestResponse;
 import net.snowflake.ingest.utils.StagedFileWrapper;
@@ -186,13 +188,20 @@ private void getHistoryAndAssertLoad(SimpleIngestManager manager, String test_fi
     Future<?> result =
         service.submit(
             () -> {
+              String startTime = Instant.ofEpochMilli(System.currentTimeMillis()).toString();
               String beginMark = null;
 
               while (true) {
 
                 try {
                   Thread.sleep(5000);
+
+                  String endTime = Instant.ofEpochMilli(System.currentTimeMillis()).toString();
                   HistoryResponse response = manager.getHistory(null, null, beginMark);
+                  HistoryRangeResponse rangeResponse =
+                      manager.getHistoryRange(null, startTime, endTime);
+
+                  assertEquals(response.files.size(), rangeResponse.files.size());
 
                   if (response != null && response.getNextBeginMark() != null) {
                     beginMark = response.getNextBeginMark();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
index c119ddd5c..767280209 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
@@ -1,28 +1,44 @@
 package net.snowflake.ingest.streaming.internal;
 
 import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CHANNEL_STATUS;
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CLIENT_CONFIGURE;
 import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
 import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.getSleepForRetryMs;
 import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.CLIENT_CONFIGURE_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
 import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils;
 import net.snowflake.client.jdbc.internal.apache.http.HttpEntity;
 import net.snowflake.client.jdbc.internal.apache.http.StatusLine;
 import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse;
 import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.ingest.TestUtils;
+import net.snowflake.ingest.connection.IngestResponseException;
+import net.snowflake.ingest.connection.JWTManager;
 import net.snowflake.ingest.connection.RequestBuilder;
+import net.snowflake.ingest.utils.HttpUtil;
+import net.snowflake.ingest.utils.SnowflakeURL;
 import org.junit.Assert;
 import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.mockito.Mockito;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
 
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({JWTManager.class})
+@PowerMockIgnore({"javax.net.ssl.*"}) /* Avoid ssl related exception from power mockito*/
 public class StreamingIngestUtilsTest {
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
@@ -108,11 +124,94 @@ public InputStream answer(InvocationOnMock invocation) throws Throwable {
             httpClient,
             requestBuilder);
 
-    Mockito.verify(requestBuilder, Mockito.times(4))
+    Mockito.verify(requestBuilder, Mockito.times(1))
         .generateStreamingIngestPostRequest(Mockito.anyString(), Mockito.any(), Mockito.any());
     Assert.assertEquals("honk", result.getMessage());
   }
 
+  @Test
+  public void testJWTRetries() throws Exception {
+    SnowflakeURL url = new SnowflakeURL(TestUtils.getAccountURL());
+    CloseableHttpClient httpClient = HttpUtil.getHttpClient(TestUtils.getAccount());
+
+    JWTManager manager =
+        new JWTManager(url.getAccount(), TestUtils.getUser(), TestUtils.getKeyPair(), null);
+    JWTManager spyManager = PowerMockito.spy(manager);
+
+    // inject spy manager
+    RequestBuilder requestBuilder =
+        PowerMockito.spy(
+            new RequestBuilder(
+                url.getAccount(),
+                TestUtils.getUser(),
+                TestUtils.getKeyPair(),
+                url.getScheme(),
+                TestUtils.getHost(),
+                url.getPort(),
+                "testJWTRetries",
+                spyManager,
+                httpClient,
+                "testJWTRetries"));
+
+    // build payload
+    Map<Object, Object> payload = new HashMap<>();
+    if (!TestUtils.getRole().isEmpty() && !TestUtils.getRole().equals("DEFAULT_ROLE")) {
+      payload.put("role", TestUtils.getRole());
+    }
+    ObjectMapper mapper = new ObjectMapper();
+
+    // request wih invalid token, should get 401 3 times
+    PowerMockito.doReturn("invalid_token").when(spyManager).getToken();
+    try {
+      ChannelsStatusResponse response =
+          executeWithRetries(
+              ChannelsStatusResponse.class,
+              CLIENT_CONFIGURE_ENDPOINT,
+              mapper.writeValueAsString(payload),
+              "client configure",
+              STREAMING_CLIENT_CONFIGURE,
+              httpClient,
+              requestBuilder);
+      Assert.fail("Expected error for invalid token");
+    } catch (SecurityException ignored) {
+    }
+    Mockito.verify(requestBuilder, Mockito.times(1)).refreshToken();
+    Mockito.clearInvocations(requestBuilder);
+
+    // request wih invalid header, should get 400 once and never refresh
+    PowerMockito.doReturn("invalid header").when(spyManager).getToken();
+    try {
+      ChannelsStatusResponse response =
+          executeWithRetries(
+              ChannelsStatusResponse.class,
+              CLIENT_CONFIGURE_ENDPOINT,
+              mapper.writeValueAsString(payload),
+              "client configure",
+              STREAMING_CLIENT_CONFIGURE,
+              httpClient,
+              requestBuilder);
+      Assert.fail("Expected error for invalid token");
+    } catch (IngestResponseException ignored) {
+    }
+    Mockito.verify(requestBuilder, Mockito.times(0)).refreshToken();
+    Mockito.clearInvocations(requestBuilder);
+
+    // request with invalid token first time but with valid token second time after refresh
+    PowerMockito.doReturn("invalid_token").doReturn(manager.getToken()).when(spyManager).getToken();
+    ChannelsStatusResponse response =
+        executeWithRetries(
+            ChannelsStatusResponse.class,
+            CLIENT_CONFIGURE_ENDPOINT,
+            mapper.writeValueAsString(payload),
+            "client configure",
+            STREAMING_CLIENT_CONFIGURE,
+            httpClient,
+            requestBuilder);
+
+    assert (response.getStatusCode() == RESPONSE_SUCCESS);
+    Mockito.verify(requestBuilder, Mockito.times(1)).refreshToken();
+  }
+
   @Test
   public void testRetriesRecovery() throws Exception {
     ChannelsStatusResponse errorResponse = new ChannelsStatusResponse();
@@ -158,7 +257,7 @@ public void testRetriesRecovery() throws Exception {
             httpClient,
             requestBuilder);
 
-    Mockito.verify(requestBuilder, Mockito.times(3))
+    Mockito.verify(requestBuilder, Mockito.times(1))
         .generateStreamingIngestPostRequest(Mockito.anyString(), Mockito.any(), Mockito.any());
 
     Assert.assertEquals("honkSuccess", result.getMessage());

From 4181badd7bbc25f84393a3498b6bbca08e7a5902 Mon Sep 17 00:00:00 2001
From: Alec Huang <alec.huang@snowflake.com>
Date: Tue, 15 Aug 2023 17:27:17 -0700
Subject: [PATCH 279/356] Move testJWTRetries test to IT (#572)

---
 .../internal/StreamingIngestUtilsIT.java      | 113 ++++++++++++++++++
 .../internal/StreamingIngestUtilsTest.java    |  99 ---------------
 2 files changed, 113 insertions(+), 99 deletions(-)
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java

diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java
new file mode 100644
index 000000000..4e054c209
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java
@@ -0,0 +1,113 @@
+package net.snowflake.ingest.streaming.internal;
+
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CLIENT_CONFIGURE;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
+import static net.snowflake.ingest.utils.Constants.CLIENT_CONFIGURE_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.HashMap;
+import java.util.Map;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
+import net.snowflake.ingest.TestUtils;
+import net.snowflake.ingest.connection.IngestResponseException;
+import net.snowflake.ingest.connection.JWTManager;
+import net.snowflake.ingest.connection.RequestBuilder;
+import net.snowflake.ingest.utils.HttpUtil;
+import net.snowflake.ingest.utils.SnowflakeURL;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({JWTManager.class})
+@PowerMockIgnore({"javax.net.ssl.*"}) /* Avoid ssl related exception from power mockito*/
+public class StreamingIngestUtilsIT {
+  @Test
+  public void testJWTRetries() throws Exception {
+    SnowflakeURL url = new SnowflakeURL(TestUtils.getAccountURL());
+    CloseableHttpClient httpClient = HttpUtil.getHttpClient(TestUtils.getAccount());
+
+    JWTManager manager =
+        new JWTManager(url.getAccount(), TestUtils.getUser(), TestUtils.getKeyPair(), null);
+    JWTManager spyManager = PowerMockito.spy(manager);
+
+    // inject spy manager
+    RequestBuilder requestBuilder =
+        PowerMockito.spy(
+            new RequestBuilder(
+                url.getAccount(),
+                TestUtils.getUser(),
+                TestUtils.getKeyPair(),
+                url.getScheme(),
+                TestUtils.getHost(),
+                url.getPort(),
+                "testJWTRetries",
+                spyManager,
+                httpClient,
+                "testJWTRetries"));
+
+    // build payload
+    Map<Object, Object> payload = new HashMap<>();
+    if (!TestUtils.getRole().isEmpty() && !TestUtils.getRole().equals("DEFAULT_ROLE")) {
+      payload.put("role", TestUtils.getRole());
+    }
+    ObjectMapper mapper = new ObjectMapper();
+
+    // request wih invalid token, should get 401 3 times
+    PowerMockito.doReturn("invalid_token").when(spyManager).getToken();
+    try {
+      ChannelsStatusResponse response =
+          executeWithRetries(
+              ChannelsStatusResponse.class,
+              CLIENT_CONFIGURE_ENDPOINT,
+              mapper.writeValueAsString(payload),
+              "client configure",
+              STREAMING_CLIENT_CONFIGURE,
+              httpClient,
+              requestBuilder);
+      Assert.fail("Expected error for invalid token");
+    } catch (SecurityException ignored) {
+    }
+    Mockito.verify(requestBuilder, Mockito.times(1)).refreshToken();
+    Mockito.clearInvocations(requestBuilder);
+
+    // request wih invalid header, should get 400 once and never refresh
+    PowerMockito.doReturn("invalid header").when(spyManager).getToken();
+    try {
+      ChannelsStatusResponse response =
+          executeWithRetries(
+              ChannelsStatusResponse.class,
+              CLIENT_CONFIGURE_ENDPOINT,
+              mapper.writeValueAsString(payload),
+              "client configure",
+              STREAMING_CLIENT_CONFIGURE,
+              httpClient,
+              requestBuilder);
+      Assert.fail("Expected error for invalid token");
+    } catch (IngestResponseException ignored) {
+    }
+    Mockito.verify(requestBuilder, Mockito.times(0)).refreshToken();
+    Mockito.clearInvocations(requestBuilder);
+
+    // request with invalid token first time but with valid token second time after refresh
+    PowerMockito.doReturn("invalid_token").doReturn(manager.getToken()).when(spyManager).getToken();
+    ChannelsStatusResponse response =
+        executeWithRetries(
+            ChannelsStatusResponse.class,
+            CLIENT_CONFIGURE_ENDPOINT,
+            mapper.writeValueAsString(payload),
+            "client configure",
+            STREAMING_CLIENT_CONFIGURE,
+            httpClient,
+            requestBuilder);
+
+    assert (response.getStatusCode() == RESPONSE_SUCCESS);
+    Mockito.verify(requestBuilder, Mockito.times(1)).refreshToken();
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
index 767280209..6aadfbcae 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java
@@ -1,44 +1,28 @@
 package net.snowflake.ingest.streaming.internal;
 
 import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CHANNEL_STATUS;
-import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CLIENT_CONFIGURE;
 import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
 import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.getSleepForRetryMs;
 import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
-import static net.snowflake.ingest.utils.Constants.CLIENT_CONFIGURE_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_ERR_GENERAL_EXCEPTION_RETRY_REQUEST;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.InputStream;
 import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
 import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils;
 import net.snowflake.client.jdbc.internal.apache.http.HttpEntity;
 import net.snowflake.client.jdbc.internal.apache.http.StatusLine;
 import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse;
 import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.ingest.TestUtils;
-import net.snowflake.ingest.connection.IngestResponseException;
-import net.snowflake.ingest.connection.JWTManager;
 import net.snowflake.ingest.connection.RequestBuilder;
-import net.snowflake.ingest.utils.HttpUtil;
-import net.snowflake.ingest.utils.SnowflakeURL;
 import org.junit.Assert;
 import org.junit.Test;
-import org.junit.runner.RunWith;
 import org.mockito.Mockito;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PowerMockIgnore;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
 
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({JWTManager.class})
-@PowerMockIgnore({"javax.net.ssl.*"}) /* Avoid ssl related exception from power mockito*/
 public class StreamingIngestUtilsTest {
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
@@ -129,89 +113,6 @@ public InputStream answer(InvocationOnMock invocation) throws Throwable {
     Assert.assertEquals("honk", result.getMessage());
   }
 
-  @Test
-  public void testJWTRetries() throws Exception {
-    SnowflakeURL url = new SnowflakeURL(TestUtils.getAccountURL());
-    CloseableHttpClient httpClient = HttpUtil.getHttpClient(TestUtils.getAccount());
-
-    JWTManager manager =
-        new JWTManager(url.getAccount(), TestUtils.getUser(), TestUtils.getKeyPair(), null);
-    JWTManager spyManager = PowerMockito.spy(manager);
-
-    // inject spy manager
-    RequestBuilder requestBuilder =
-        PowerMockito.spy(
-            new RequestBuilder(
-                url.getAccount(),
-                TestUtils.getUser(),
-                TestUtils.getKeyPair(),
-                url.getScheme(),
-                TestUtils.getHost(),
-                url.getPort(),
-                "testJWTRetries",
-                spyManager,
-                httpClient,
-                "testJWTRetries"));
-
-    // build payload
-    Map<Object, Object> payload = new HashMap<>();
-    if (!TestUtils.getRole().isEmpty() && !TestUtils.getRole().equals("DEFAULT_ROLE")) {
-      payload.put("role", TestUtils.getRole());
-    }
-    ObjectMapper mapper = new ObjectMapper();
-
-    // request wih invalid token, should get 401 3 times
-    PowerMockito.doReturn("invalid_token").when(spyManager).getToken();
-    try {
-      ChannelsStatusResponse response =
-          executeWithRetries(
-              ChannelsStatusResponse.class,
-              CLIENT_CONFIGURE_ENDPOINT,
-              mapper.writeValueAsString(payload),
-              "client configure",
-              STREAMING_CLIENT_CONFIGURE,
-              httpClient,
-              requestBuilder);
-      Assert.fail("Expected error for invalid token");
-    } catch (SecurityException ignored) {
-    }
-    Mockito.verify(requestBuilder, Mockito.times(1)).refreshToken();
-    Mockito.clearInvocations(requestBuilder);
-
-    // request wih invalid header, should get 400 once and never refresh
-    PowerMockito.doReturn("invalid header").when(spyManager).getToken();
-    try {
-      ChannelsStatusResponse response =
-          executeWithRetries(
-              ChannelsStatusResponse.class,
-              CLIENT_CONFIGURE_ENDPOINT,
-              mapper.writeValueAsString(payload),
-              "client configure",
-              STREAMING_CLIENT_CONFIGURE,
-              httpClient,
-              requestBuilder);
-      Assert.fail("Expected error for invalid token");
-    } catch (IngestResponseException ignored) {
-    }
-    Mockito.verify(requestBuilder, Mockito.times(0)).refreshToken();
-    Mockito.clearInvocations(requestBuilder);
-
-    // request with invalid token first time but with valid token second time after refresh
-    PowerMockito.doReturn("invalid_token").doReturn(manager.getToken()).when(spyManager).getToken();
-    ChannelsStatusResponse response =
-        executeWithRetries(
-            ChannelsStatusResponse.class,
-            CLIENT_CONFIGURE_ENDPOINT,
-            mapper.writeValueAsString(payload),
-            "client configure",
-            STREAMING_CLIENT_CONFIGURE,
-            httpClient,
-            requestBuilder);
-
-    assert (response.getStatusCode() == RESPONSE_SUCCESS);
-    Mockito.verify(requestBuilder, Mockito.times(1)).refreshToken();
-  }
-
   @Test
   public void testRetriesRecovery() throws Exception {
     ChannelsStatusResponse errorResponse = new ChannelsStatusResponse();

From 290a1680bba1b7c54f2a665b5c21f7baea9929b9 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Thu, 17 Aug 2023 16:08:18 +0200
Subject: [PATCH 280/356] SNOW-861190 Add warning for large batches in
 insertRows (#551)

---
 .../streaming/internal/AbstractRowBuffer.java | 34 ++++++++++++++-
 .../streaming/internal/ParquetRowBuffer.java  |  3 +-
 ...owflakeStreamingIngestChannelInternal.java |  4 +-
 .../net/snowflake/ingest/utils/ErrorCode.java |  3 +-
 .../ingest/ingest_error_messages.properties   |  1 +
 .../streaming/internal/RowBufferTest.java     | 43 +++++++++++++++++--
 6 files changed, 79 insertions(+), 9 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index e9addc4a9..94e885e8a 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -38,6 +38,9 @@ abstract class AbstractRowBuffer<T> implements RowBuffer<T> {
   // side scanner
   private static final int INVALID_SERVER_SIDE_DATA_TYPE_ORDINAL = -1;
 
+  // The maximum recommended batch size of data passed into a single insertRows() call
+  private static final int INSERT_ROWS_RECOMMENDED_MAX_BATCH_SIZE_IN_BYTES = 16 * 1024 * 1024;
+
   // Snowflake table column logical type
   enum ColumnLogicalType {
     ANY,
@@ -159,6 +162,8 @@ public int getOrdinal() {
   // Metric callback to report size of inserted rows
   private final Consumer<Float> rowSizeMetric;
 
+  private final long maxChunkSizeInBytes;
+
   // State of the owning channel
   final ChannelRuntimeState channelState;
 
@@ -172,7 +177,8 @@ public int getOrdinal() {
       ZoneId defaultTimezone,
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
-      ChannelRuntimeState channelRuntimeState) {
+      ChannelRuntimeState channelRuntimeState,
+      long maxChunkSizeInBytes) {
     this.onErrorOption = onErrorOption;
     this.defaultTimezone = defaultTimezone;
     this.rowSizeMetric = rowSizeMetric;
@@ -182,6 +188,7 @@ public int getOrdinal() {
     this.flushLock = new ReentrantLock();
     this.bufferedRowCount = 0;
     this.bufferSize = 0F;
+    this.maxChunkSizeInBytes = maxChunkSizeInBytes;
 
     // Initialize empty stats
     this.statsMap = new HashMap<>();
@@ -313,11 +320,13 @@ public InsertValidationResponse insertRows(
             error.setException(new SFException(e, ErrorCode.INTERNAL_ERROR, e.getMessage()));
             response.addError(error);
           }
+          checkBatchSizeEnforcedMaximum(rowsSizeInBytes);
           rowIndex++;
           if (this.bufferedRowCount == Integer.MAX_VALUE) {
             throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
           }
         }
+        checkBatchSizeRecommendedMaximum(rowsSizeInBytes);
       } else {
         // If the on_error option is ABORT, simply throw the first exception
         float tempRowsSizeInBytes = 0F;
@@ -326,8 +335,10 @@ public InsertValidationResponse insertRows(
           Set<String> inputColumnNames = verifyInputColumns(row, null);
           tempRowsSizeInBytes +=
               addTempRow(row, tempRowCount, this.tempStatsMap, inputColumnNames, tempRowCount);
+          checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes);
           tempRowCount++;
         }
+        checkBatchSizeRecommendedMaximum(tempRowsSizeInBytes);
 
         moveTempRowsToActualBuffer(tempRowCount);
 
@@ -549,4 +560,25 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
             ErrorCode.INTERNAL_ERROR, "Unsupported BDEC format version: " + bdecVersion);
     }
   }
+
+  private void checkBatchSizeEnforcedMaximum(float batchSizeInBytes) {
+    if (batchSizeInBytes > maxChunkSizeInBytes) {
+      throw new SFException(
+          ErrorCode.MAX_BATCH_SIZE_EXCEEDED,
+          maxChunkSizeInBytes,
+          INSERT_ROWS_RECOMMENDED_MAX_BATCH_SIZE_IN_BYTES);
+    }
+  }
+
+  private void checkBatchSizeRecommendedMaximum(float batchSizeInBytes) {
+    if (batchSizeInBytes > INSERT_ROWS_RECOMMENDED_MAX_BATCH_SIZE_IN_BYTES) {
+      logger.logWarn(
+          "The batch of rows passed to 'insertRows' is over the recommended max batch size. Given"
+              + " {} bytes, recommended max batch is {} bytes. For optimal performance and memory"
+              + " utilization, we recommend splitting large batches into multiple smaller ones and"
+              + " call insertRows for each smaller batch separately.",
+          batchSizeInBytes,
+          INSERT_ROWS_RECOMMENDED_MAX_BATCH_SIZE_IN_BYTES);
+    }
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 99fd25fb0..94baaf384 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -70,7 +70,8 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
         defaultTimezone,
         fullyQualifiedChannelName,
         rowSizeMetric,
-        channelRuntimeState);
+        channelRuntimeState,
+        maxChunkSizeInBytes);
     this.fieldIndex = new HashMap<>();
     this.metadata = new HashMap<>();
     this.data = new ArrayList<>();
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 5ddc654ab..07f64cc58 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -126,8 +126,8 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
                 ? owningClient.getParameterProvider().getEnableParquetInternalBuffering()
                 : ParameterProvider.ENABLE_PARQUET_INTERNAL_BUFFERING_DEFAULT,
             owningClient != null
-                ? owningClient.getParameterProvider().getMaxChannelSizeInBytes()
-                : ParameterProvider.MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT,
+                ? owningClient.getParameterProvider().getMaxChunkSizeInBytes()
+                : ParameterProvider.MAX_CHUNK_SIZE_IN_BYTES_DEFAULT,
             owningClient != null
                 ? owningClient.getParameterProvider().getMaxAllowedRowSizeInBytes()
                 : ParameterProvider.MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT);
diff --git a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
index cae9273f4..97bdae0f1 100644
--- a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
+++ b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
@@ -39,7 +39,8 @@ public enum ErrorCode {
   MAX_ROW_SIZE_EXCEEDED("0031"),
   MAKE_URI_FAILURE("0032"),
   OAUTH_REFRESH_TOKEN_ERROR("0033"),
-  INVALID_CONFIG_PARAMETER("0034");
+  INVALID_CONFIG_PARAMETER("0034"),
+  MAX_BATCH_SIZE_EXCEEDED("0035");
 
   public static final String errorMessageResource = "net.snowflake.ingest.ingest_error_messages";
 
diff --git a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
index 134c3dac5..9698799f7 100644
--- a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
+++ b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
@@ -37,3 +37,4 @@
 0032=URI builder fail to build url: {0}
 0033=OAuth token refresh failure: {0}
 0034=Invalid config parameter: {0}
+0035=Too large batch of rows passed to insertRows, the batch size cannot exceed {0} bytes, recommended batch size for optimal performance and memory utilization is {1} bytes. We recommend splitting large batches into multiple smaller ones and call insertRows for each smaller batch separately.
\ No newline at end of file
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 7fa598647..49cf3dce8 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -2,7 +2,7 @@
 
 import static java.time.ZoneOffset.UTC;
 import static net.snowflake.ingest.utils.ParameterProvider.MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT;
-import static net.snowflake.ingest.utils.ParameterProvider.MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT;
+import static net.snowflake.ingest.utils.ParameterProvider.MAX_CHUNK_SIZE_IN_BYTES_DEFAULT;
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
@@ -116,14 +116,12 @@ private AbstractRowBuffer<?> createTestBuffer(OpenChannelRequest.OnErrorOption o
         rs -> {},
         initialState,
         enableParquetMemoryOptimization,
-        MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT,
+        MAX_CHUNK_SIZE_IN_BYTES_DEFAULT,
         MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT);
   }
 
   @Test
   public void testCollatedColumnsAreRejected() {
-    AbstractRowBuffer<?> testBuffer = createTestBuffer(OpenChannelRequest.OnErrorOption.ABORT);
-
     ColumnMetadata collatedColumn = new ColumnMetadata();
     collatedColumn.setName("COLCHAR");
     collatedColumn.setPhysicalType("LOB");
@@ -937,6 +935,43 @@ private void testE2ETimeHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
     Assert.assertEquals(1, result.getColumnEps().get("COLTIMESB8").getCurrentNullCount());
   }
 
+  @Test
+  public void testMaxInsertRowsBatchSize() {
+    testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+    testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption.ABORT);
+  }
+
+  private void testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
+    AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
+    ColumnMetadata colBinary = new ColumnMetadata();
+    colBinary.setName("COLBINARY");
+    colBinary.setPhysicalType("LOB");
+    colBinary.setNullable(true);
+    colBinary.setLogicalType("BINARY");
+    colBinary.setLength(8 * 1024 * 1024);
+    colBinary.setByteLength(8 * 1024 * 1024);
+
+    byte[] arr = new byte[8 * 1024 * 1024];
+    innerBuffer.setupSchema(Collections.singletonList(colBinary));
+    List<Map<String, Object>> rows = new ArrayList<>();
+    for (int i = 0; i < 15; i++) {
+      rows.add(Collections.singletonMap("COLBINARY", arr));
+    }
+
+    // Insert rows should succeed
+    innerBuffer.insertRows(rows, "");
+
+    // After adding another row, it should fail due to too large batch of rows passed to
+    // insertRows() in one go
+    rows.add(Collections.singletonMap("COLBINARY", arr));
+    try {
+      innerBuffer.insertRows(rows, "");
+      Assert.fail("Inserting rows should have failed");
+    } catch (SFException e) {
+      Assert.assertEquals(ErrorCode.MAX_BATCH_SIZE_EXCEEDED.getMessageCode(), e.getVendorCode());
+    }
+  }
+
   @Test
   public void testNullableCheck() {
     testNullableCheckHelper(OpenChannelRequest.OnErrorOption.ABORT);

From 238b65d0afb54997da7ac71b8738bb097e2d23cb Mon Sep 17 00:00:00 2001
From: Dhara Shah <dhara.shah@snowflake.com>
Date: Fri, 18 Aug 2023 16:20:31 -0700
Subject: [PATCH 281/356] Using jenkins sonatype username and password instead
 of my own (#571)

* Using jenkins sonatype user and password instead of my own

* skipping test

* using jenkins sonatype username and password instead of user credential

* removed dskiptests=true

* updating sonatype username and password to unshaded_snapshot_deploy file

* reverted it to original

* removed extra spaces
---
 snapshot_deploy.sh          | 4 ++--
 unshaded_snapshot_deploy.sh | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/snapshot_deploy.sh b/snapshot_deploy.sh
index 05fbaf4e3..78a9bdc90 100755
--- a/snapshot_deploy.sh
+++ b/snapshot_deploy.sh
@@ -2,8 +2,8 @@
 
 THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 export GPG_KEY_ID="Snowflake Computing"
-export LDAP_USER="$sonatype_user"
-export LDAP_PWD="$sonatype_password"
+export LDAP_USER="$INTERNAL_NEXUS_USERNAME"
+export LDAP_PWD="$INTERNAL_NEXUS_PASSWORD"
 
 if [ -z "$GPG_KEY_PASSPHRASE" ]; then
   echo "[ERROR] GPG passphrase is not specified for $GPG_KEY_ID!"
diff --git a/unshaded_snapshot_deploy.sh b/unshaded_snapshot_deploy.sh
index dcd578218..a27911e4b 100755
--- a/unshaded_snapshot_deploy.sh
+++ b/unshaded_snapshot_deploy.sh
@@ -3,8 +3,8 @@
 THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 export GPG_KEY_ID="Snowflake Computing"
-export LDAP_USER="$sonatype_user"
-export LDAP_PWD="$sonatype_password"
+export LDAP_USER="$INTERNAL_NEXUS_USERNAME"
+export LDAP_PWD="$INTERNAL_NEXUS_PASSWORD"
 
 if [ -z "$GPG_KEY_PASSPHRASE" ]; then
   echo "[ERROR] GPG passphrase is not specified for $GPG_KEY_ID!"

From d66d1a7009378fa2810a63a955c3ff2996b91d0e Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 24 Aug 2023 12:53:51 -0700
Subject: [PATCH 282/356] NO-SNOW: Add a few requested changes (#576)

Doing it manually for now given that we want to do a release soon, we need to look into how it can be done automatically
---
 .../SnowflakeStreamingIngestChannel.java      |  1 +
 .../SnowflakeStreamingIngestClient.java       | 11 +++
 .../streaming/internal/FlushService.java      | 74 +++++++++----------
 .../streaming/internal/RegisterService.java   | 12 ++-
 ...nowflakeStreamingIngestClientInternal.java | 24 ++++++
 .../streaming/internal/FlushServiceTest.java  |  2 +-
 .../internal/RegisterServiceTest.java         |  6 +-
 .../SnowflakeStreamingIngestClientTest.java   | 65 ++++++++++++++++
 8 files changed, 147 insertions(+), 48 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
index ce576372e..5fdadc90b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
@@ -251,5 +251,6 @@ InsertValidationResponse insertRows(
    *
    * @return the latest committed offset token
    */
+  @Nullable
   String getLatestCommittedOffsetToken();
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
index 6921bcabc..3ae4a83d1 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
@@ -4,6 +4,9 @@
 
 package net.snowflake.ingest.streaming;
 
+import java.util.List;
+import java.util.Map;
+
 /**
  * A class that is the starting point for using the Streaming Ingest client APIs, a single client
  * maps to exactly one account in Snowflake; however, multiple clients can point to the same
@@ -45,4 +48,12 @@ public interface SnowflakeStreamingIngestClient extends AutoCloseable {
    * @return a boolean to indicate whether the client is closed
    */
   boolean isClosed();
+
+  /**
+   * Return the latest committed offset token for a list of channels
+   *
+   * @return a map of channel fully qualified name to latest committed offset token
+   */
+  Map<String, String> getLatestCommittedOffsetTokens(
+      List<SnowflakeStreamingIngestChannel> channels);
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 86358b24d..8016f9b5c 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -63,16 +63,16 @@ class FlushService<T> {
   // Static class to save the list of channels that are used to build a blob, which is mainly used
   // to invalidate all the channels when there is a failure
   static class BlobData<T> {
-    private final String filePath;
+    private final String path;
     private final List<List<ChannelData<T>>> data;
 
-    BlobData(String filePath, List<List<ChannelData<T>>> data) {
-      this.filePath = filePath;
+    BlobData(String path, List<List<ChannelData<T>>> data) {
+      this.path = path;
       this.data = data;
     }
 
-    String getFilePath() {
-      return filePath;
+    String getPath() {
+      return path;
     }
 
     List<List<ChannelData<T>>> getData() {
@@ -118,7 +118,7 @@ List<List<ChannelData<T>>> getData() {
   // A map which stores the timer for each blob in order to capture the flush latency
   private final Map<String, Timer.Context> latencyTimerContextMap;
 
-  // blob file version
+  // blob encoding version
   private final Constants.BdecVersion bdecVersion;
 
   /**
@@ -342,7 +342,7 @@ void distributeFlushTasks() {
     while (itr.hasNext() || !leftoverChannelsDataPerTable.isEmpty()) {
       List<List<ChannelData<T>>> blobData = new ArrayList<>();
       float totalBufferSizeInBytes = 0F;
-      final String filePath = getFilePath(this.targetStage.getClientPrefix());
+      final String blobPath = getBlobPath(this.targetStage.getClientPrefix());
 
       // Distribute work at table level, split the blob if reaching the blob size limit or the
       // channel has different encryption key ids
@@ -360,7 +360,7 @@ void distributeFlushTasks() {
               .forEach(
                   channel -> {
                     if (channel.isValid()) {
-                      ChannelData<T> data = channel.getData(filePath);
+                      ChannelData<T> data = channel.getData(blobPath);
                       if (data != null) {
                         channelsDataPerTable.add(data);
                       }
@@ -385,7 +385,7 @@ && shouldStopProcessing(
               logger.logInfo(
                   "Creation of another blob is needed because of blob/chunk size limit or"
                       + " different encryption ids or different schema, client={}, table={},"
-                      + " fileSize={}, chunkSize={}, nextChannelSize={}, encryptionId1={},"
+                      + " blobSize={}, chunkSize={}, nextChannelSize={}, encryptionId1={},"
                       + " encryptionId2={}, schema1={}, schema2={}",
                   this.owningClient.getName(),
                   channelData.getChannelContext().getTableName(),
@@ -412,32 +412,32 @@ && shouldStopProcessing(
 
       // Kick off a build job
       if (blobData.isEmpty()) {
-        // we decrement the counter so that we do not have gaps in the filenames created by this
-        // client. See method getFilePath() below.
+        // we decrement the counter so that we do not have gaps in the blob names created by this
+        // client. See method getBlobPath() below.
         this.counter.decrementAndGet();
       } else {
         long flushStartMs = System.currentTimeMillis();
         if (this.owningClient.flushLatency != null) {
-          latencyTimerContextMap.putIfAbsent(filePath, this.owningClient.flushLatency.time());
+          latencyTimerContextMap.putIfAbsent(blobPath, this.owningClient.flushLatency.time());
         }
         blobs.add(
             new Pair<>(
-                new BlobData<>(filePath, blobData),
+                new BlobData<>(blobPath, blobData),
                 CompletableFuture.supplyAsync(
                     () -> {
                       try {
-                        BlobMetadata blobMetadata = buildAndUpload(filePath, blobData);
+                        BlobMetadata blobMetadata = buildAndUpload(blobPath, blobData);
                         blobMetadata.getBlobStats().setFlushStartMs(flushStartMs);
                         return blobMetadata;
                       } catch (Throwable e) {
                         Throwable ex = e.getCause() == null ? e : e.getCause();
                         String errorMessage =
                             String.format(
-                                "Building blob failed, client=%s, file=%s, exception=%s,"
+                                "Building blob failed, client=%s, blob=%s, exception=%s,"
                                     + " detail=%s, trace=%s, all channels in the blob will be"
                                     + " invalidated",
                                 this.owningClient.getName(),
-                                filePath,
+                                blobPath,
                                 ex,
                                 ex.getMessage(),
                                 getStackTrace(ex));
@@ -468,7 +468,7 @@ && shouldStopProcessing(
         logger.logInfo(
             "buildAndUpload task added for client={}, blob={}, buildUploadWorkers stats={}",
             this.owningClient.getName(),
-            filePath,
+            blobPath,
             this.buildUploadWorkers.toString());
       }
     }
@@ -481,7 +481,7 @@ && shouldStopProcessing(
    * Check whether we should stop merging more channels into the same chunk, we need to stop in a
    * few cases:
    *
-   * <p>When the file size is larger than a certain threshold
+   * <p>When the blob size is larger than a certain threshold
    *
    * <p>When the chunk size is larger than a certain threshold
    *
@@ -504,44 +504,44 @@ private boolean shouldStopProcessing(
   }
 
   /**
-   * Builds and uploads file to cloud storage.
+   * Builds and uploads blob to cloud storage.
    *
-   * @param filePath Path of the destination file in cloud storage
+   * @param blobPath Path of the destination blob in cloud storage
    * @param blobData All the data for one blob. Assumes that all ChannelData in the inner List
    *     belongs to the same table. Will error if this is not the case
    * @return BlobMetadata for FlushService.upload
    */
-  BlobMetadata buildAndUpload(String filePath, List<List<ChannelData<T>>> blobData)
+  BlobMetadata buildAndUpload(String blobPath, List<List<ChannelData<T>>> blobData)
       throws IOException, NoSuchAlgorithmException, InvalidAlgorithmParameterException,
           NoSuchPaddingException, IllegalBlockSizeException, BadPaddingException,
           InvalidKeyException {
     Timer.Context buildContext = Utils.createTimerContext(this.owningClient.buildLatency);
 
     // Construct the blob along with the metadata of the blob
-    BlobBuilder.Blob blob = BlobBuilder.constructBlobAndMetadata(filePath, blobData, bdecVersion);
+    BlobBuilder.Blob blob = BlobBuilder.constructBlobAndMetadata(blobPath, blobData, bdecVersion);
 
     blob.blobStats.setBuildDurationMs(buildContext);
 
-    return upload(filePath, blob.blobBytes, blob.chunksMetadataList, blob.blobStats);
+    return upload(blobPath, blob.blobBytes, blob.chunksMetadataList, blob.blobStats);
   }
 
   /**
    * Upload a blob to Streaming Ingest dedicated stage
    *
-   * @param filePath full path of the blob file
+   * @param blobPath full path of the blob
    * @param blob blob data
    * @param metadata a list of chunk metadata
    * @param blobStats an object to track latencies and other stats of the blob
    * @return BlobMetadata object used to create the register blob request
    */
   BlobMetadata upload(
-      String filePath, byte[] blob, List<ChunkMetadata> metadata, BlobStats blobStats)
+      String blobPath, byte[] blob, List<ChunkMetadata> metadata, BlobStats blobStats)
       throws NoSuchAlgorithmException {
-    logger.logInfo("Start uploading file={}, size={}", filePath, blob.length);
+    logger.logInfo("Start uploading blob={}, size={}", blobPath, blob.length);
     long startTime = System.currentTimeMillis();
 
     Timer.Context uploadContext = Utils.createTimerContext(this.owningClient.uploadLatency);
-    this.targetStage.put(filePath, blob);
+    this.targetStage.put(blobPath, blob);
 
     if (uploadContext != null) {
       blobStats.setUploadDurationMs(uploadContext);
@@ -552,13 +552,13 @@ BlobMetadata upload(
     }
 
     logger.logInfo(
-        "Finish uploading file={}, size={}, timeInMillis={}",
-        filePath,
+        "Finish uploading blob={}, size={}, timeInMillis={}",
+        blobPath,
         blob.length,
         System.currentTimeMillis() - startTime);
 
     return BlobMetadata.createBlobMetadata(
-        filePath, BlobBuilder.computeMD5(blob), bdecVersion, metadata, blobStats);
+        blobPath, BlobBuilder.computeMD5(blob), bdecVersion, metadata, blobStats);
   }
 
   /**
@@ -592,18 +592,18 @@ void setNeedFlush() {
   }
 
   /**
-   * Generate a blob file path, which is: "YEAR/MONTH/DAY_OF_MONTH/HOUR_OF_DAY/MINUTE/<current utc
+   * Generate a blob path, which is: "YEAR/MONTH/DAY_OF_MONTH/HOUR_OF_DAY/MINUTE/<current utc
    * timestamp + client unique prefix + thread id + counter>.BDEC"
    *
    * @return the generated blob file path
    */
-  private String getFilePath(String clientPrefix) {
+  private String getBlobPath(String clientPrefix) {
     Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
-    return getFilePath(calendar, clientPrefix);
+    return getBlobPath(calendar, clientPrefix);
   }
 
   /** For TESTING */
-  String getFilePath(Calendar calendar, String clientPrefix) {
+  String getBlobPath(Calendar calendar, String clientPrefix) {
     if (isTestMode && clientPrefix == null) {
       clientPrefix = "testPrefix";
     }
@@ -616,8 +616,8 @@ String getFilePath(Calendar calendar, String clientPrefix) {
     int minute = calendar.get(Calendar.MINUTE);
     long time = TimeUnit.MILLISECONDS.toSeconds(calendar.getTimeInMillis());
     long threadId = Thread.currentThread().getId();
-    // Create the file short name, the clientPrefix contains the deployment id
-    String fileShortName =
+    // Create the blob short name, the clientPrefix contains the deployment id
+    String blobShortName =
         Long.toString(time, 36)
             + "_"
             + clientPrefix
@@ -627,7 +627,7 @@ String getFilePath(Calendar calendar, String clientPrefix) {
             + this.counter.getAndIncrement()
             + "."
             + BLOB_EXTENSION_TYPE;
-    return year + "/" + month + "/" + day + "/" + hour + "/" + minute + "/" + fileShortName;
+    return year + "/" + month + "/" + day + "/" + hour + "/" + minute + "/" + blobShortName;
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
index 1e3e63d98..0be99b821 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RegisterService.java
@@ -112,7 +112,7 @@ List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyT
         int retry = 0;
         logger.logDebug(
             "Start loop outer for uploading blobs={}",
-            oldList.stream().map(blob -> blob.getKey().getFilePath()).collect(Collectors.toList()));
+            oldList.stream().map(blob -> blob.getKey().getPath()).collect(Collectors.toList()));
         while (idx < oldList.size()) {
           List<BlobMetadata> blobs = new ArrayList<>();
           long startTime = System.currentTimeMillis();
@@ -125,15 +125,13 @@ List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyT
                 oldList.get(idx);
             try {
               logger.logDebug(
-                  "Start waiting on uploading blob={}, idx={}",
-                  futureBlob.getKey().getFilePath(),
-                  idx);
+                  "Start waiting on uploading blob={}, idx={}", futureBlob.getKey().getPath(), idx);
               // Wait for uploading to finish
               BlobMetadata blob =
                   futureBlob.getValue().get(BLOB_UPLOAD_TIMEOUT_IN_SEC, TimeUnit.SECONDS);
               logger.logDebug(
                   "Finish waiting on uploading blob={}, idx={}",
-                  futureBlob.getKey().getFilePath(),
+                  futureBlob.getKey().getPath(),
                   idx);
               if (blob != null) {
                 blobs.add(blob);
@@ -152,7 +150,7 @@ List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyT
                       < this.owningClient.getParameterProvider().getBlobUploadMaxRetryCount()) {
                 logger.logInfo(
                     "Retry on waiting for uploading blob={}, idx={}",
-                    futureBlob.getKey().getFilePath(),
+                    futureBlob.getKey().getPath(),
                     idx);
                 retry++;
                 break;
@@ -163,7 +161,7 @@ List<FlushService.BlobData<T>> registerBlobs(Map<String, Timer.Context> latencyT
                           + " detail=%s, cause=%s, cause_detail=%s, cause_trace=%s all channels in"
                           + " the blob will be invalidated",
                       this.owningClient.getName(),
-                      futureBlob.getKey().getFilePath(),
+                      futureBlob.getKey().getPath(),
                       e,
                       e.getMessage(),
                       e.getCause(),
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 7836e7b44..5916e472d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -64,6 +64,7 @@
 import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.connection.TelemetryService;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
@@ -362,6 +363,29 @@ public SnowflakeStreamingIngestChannelInternal<?> openChannel(OpenChannelRequest
     }
   }
 
+  /**
+   * Return the latest committed/persisted offset token for all channels
+   *
+   * @return map of channel to the latest persisted offset token
+   */
+  @Override
+  public Map<String, String> getLatestCommittedOffsetTokens(
+      List<SnowflakeStreamingIngestChannel> channels) {
+    List<SnowflakeStreamingIngestChannelInternal<?>> internalChannels =
+        channels.stream()
+            .map(c -> (SnowflakeStreamingIngestChannelInternal<?>) c)
+            .collect(Collectors.toList());
+    List<ChannelsStatusResponse.ChannelStatusResponseDTO> channelsStatus =
+        getChannelsStatus(internalChannels).getChannels();
+    Map<String, String> result = new HashMap<>();
+    for (int idx = 0; idx < channels.size(); idx++) {
+      result.put(
+          channels.get(idx).getFullyQualifiedName(),
+          channelsStatus.get(idx).getPersistedOffsetToken());
+    }
+    return result;
+  }
+
   /**
    * Fetch channels status from Snowflake
    *
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index c8de18f77..0b8e8b4cd 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -379,7 +379,7 @@ public void testGetFilePath() {
     FlushService<?> flushService = testContext.flushService;
     Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
     String clientPrefix = "honk";
-    String outputString = flushService.getFilePath(calendar, clientPrefix);
+    String outputString = flushService.getBlobPath(calendar, clientPrefix);
     Path outputPath = Paths.get(outputString);
     Assert.assertTrue(outputPath.getFileName().toString().contains(clientPrefix));
     Assert.assertTrue(
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
index 9638401d3..000b948e9 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
@@ -60,7 +60,7 @@ public void testRegisterServiceTimeoutException() throws Exception {
       List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null);
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
-      Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
+      Assert.assertEquals("fail", errorBlobs.get(0).getPath());
     } catch (Exception e) {
       Assert.fail("The timeout exception should be caught in registerBlobs");
     }
@@ -96,7 +96,7 @@ public void testRegisterServiceTimeoutException_testRetries() throws Exception {
       List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null);
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
-      Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
+      Assert.assertEquals("fail", errorBlobs.get(0).getPath());
     } catch (Exception e) {
       Assert.fail("The timeout exception should be caught in registerBlobs");
     }
@@ -118,7 +118,7 @@ public void testRegisterServiceNonTimeoutException() {
       List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null);
       Assert.assertEquals(0, rs.getBlobsList().size());
       Assert.assertEquals(1, errorBlobs.size());
-      Assert.assertEquals("fail", errorBlobs.get(0).getFilePath());
+      Assert.assertEquals("fail", errorBlobs.get(0).getPath());
     } catch (Exception e) {
       Assert.fail("The exception should be caught in registerBlobs");
     }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index a134e17ea..ec9f671bb 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -1235,4 +1235,69 @@ public void testFlushServiceException() throws Exception {
             requestBuilder,
             parameterMap);
   }
+
+  @Test
+  public void testGetLatestCommittedOffsetTokens() throws Exception {
+    ChannelsStatusResponse.ChannelStatusResponseDTO channelStatus =
+        new ChannelsStatusResponse.ChannelStatusResponseDTO();
+    channelStatus.setStatusCode(RESPONSE_SUCCESS);
+    channelStatus.setPersistedOffsetToken("foo");
+    ChannelsStatusResponse response = new ChannelsStatusResponse();
+    response.setStatusCode(0L);
+    response.setMessage("honk");
+    response.setChannels(Collections.singletonList(channelStatus));
+    String responseString = objectMapper.writeValueAsString(response);
+
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
+    StatusLine statusLine = Mockito.mock(StatusLine.class);
+    HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
+    Mockito.when(statusLine.getStatusCode()).thenReturn(200);
+    Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    Mockito.when(httpResponse.getEntity()).thenReturn(httpEntity);
+    Mockito.when(httpEntity.getContent()).thenReturn(IOUtils.toInputStream(responseString));
+    Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
+
+    RequestBuilder requestBuilder =
+        Mockito.spy(
+            new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair()));
+    SnowflakeStreamingIngestClientInternal<?> client =
+        new SnowflakeStreamingIngestClientInternal<>(
+            "client",
+            new SnowflakeURL("snowflake.dev.local:8082"),
+            null,
+            httpClient,
+            true,
+            requestBuilder,
+            null);
+
+    SnowflakeStreamingIngestChannelInternal<?> channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
+            "channel",
+            "db",
+            "schemaName",
+            "tableName",
+            "0",
+            0L,
+            0L,
+            null,
+            "key",
+            1234L,
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            ZoneOffset.UTC,
+            BDEC_VERSION);
+
+    ChannelsStatusRequest.ChannelStatusRequestDTO dto =
+        new ChannelsStatusRequest.ChannelStatusRequestDTO(channel);
+    ChannelsStatusRequest request = new ChannelsStatusRequest();
+    request.setRequestId("null_0");
+    request.setChannels(Collections.singletonList(dto));
+    Map<String, String> result =
+        client.getLatestCommittedOffsetTokens(Collections.singletonList(channel));
+    Assert.assertEquals(
+        channelStatus.getPersistedOffsetToken(), result.get(channel.getFullyQualifiedName()));
+    Mockito.verify(requestBuilder)
+        .generateStreamingIngestPostRequest(
+            objectMapper.writeValueAsString(request), CHANNEL_STATUS_ENDPOINT, "channel status");
+  }
 }

From aeda5fb6d4d43691982c70d5e1b2f36bdda4f30b Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 31 Aug 2023 16:51:04 -0700
Subject: [PATCH 283/356] V2.0.3 release (#577)

V2.0.3 release
---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 92e544855..360e70c38 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.0.3-SNAPSHOT</version>
+  <version>2.0.3</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index ffe2fff13..a8018954c 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.0.3-SNAPSHOT";
+  public static final String DEFAULT_VERSION = "2.0.3";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From f70f5bfb0ba99f529e06881e9db7345b6aa48bb2 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Mon, 4 Sep 2023 14:28:24 +0200
Subject: [PATCH 284/356] Remove Public Preview note (#582)

Remove a note about Snowpipe Streaming being in public preview.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ba4a1f813..edbded16d 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@ The Snowflake Ingest Service SDK allows users to ingest files into their
 Snowflake data warehouse in a programmatic fashion via key-pair
 authentication. Currently, we support ingestion through the following APIs:
 1. [Snowpipe](https://docs.snowflake.com/en/user-guide/data-load-snowpipe-rest-gs.html#client-requirement-java-or-python-sdk)
-2. [Snowpipe Streaming](https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview) - Under Public Preview
+2. [Snowpipe Streaming](https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview)
 
 # Dependencies
 

From 71cd90f198437f9cee50139efaad46d2f42ef920 Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Fri, 8 Sep 2023 10:31:14 +0200
Subject: [PATCH 285/356] @no-snow refactor a few variables into
 ClientBufferParameters (#581)

---
 .../streaming/internal/AbstractRowBuffer.java | 21 +++--
 .../internal/ClientBufferParameters.java      | 77 +++++++++++++++++++
 .../streaming/internal/ParquetRowBuffer.java  | 35 +++++----
 ...owflakeStreamingIngestChannelInternal.java | 11 +--
 .../streaming/internal/RowBufferTest.java     |  7 +-
 5 files changed, 109 insertions(+), 42 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ClientBufferParameters.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 94e885e8a..fa2c65006 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -162,8 +162,6 @@ public int getOrdinal() {
   // Metric callback to report size of inserted rows
   private final Consumer<Float> rowSizeMetric;
 
-  private final long maxChunkSizeInBytes;
-
   // State of the owning channel
   final ChannelRuntimeState channelState;
 
@@ -172,13 +170,16 @@ public int getOrdinal() {
 
   final ZoneId defaultTimezone;
 
+  // Buffer parameters that are set at the owning client level
+  final ClientBufferParameters clientBufferParameters;
+
   AbstractRowBuffer(
       OpenChannelRequest.OnErrorOption onErrorOption,
       ZoneId defaultTimezone,
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelRuntimeState,
-      long maxChunkSizeInBytes) {
+      ClientBufferParameters clientBufferParameters) {
     this.onErrorOption = onErrorOption;
     this.defaultTimezone = defaultTimezone;
     this.rowSizeMetric = rowSizeMetric;
@@ -188,7 +189,7 @@ public int getOrdinal() {
     this.flushLock = new ReentrantLock();
     this.bufferedRowCount = 0;
     this.bufferSize = 0F;
-    this.maxChunkSizeInBytes = maxChunkSizeInBytes;
+    this.clientBufferParameters = clientBufferParameters;
 
     // Initialize empty stats
     this.statsMap = new HashMap<>();
@@ -539,9 +540,7 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelRuntimeState,
-      boolean enableParquetMemoryOptimization,
-      long maxChunkSizeInBytes,
-      long maxRowSizeInBytes) {
+      ClientBufferParameters clientBufferParameters) {
     switch (bdecVersion) {
       case THREE:
         //noinspection unchecked
@@ -552,9 +551,7 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
                 fullyQualifiedChannelName,
                 rowSizeMetric,
                 channelRuntimeState,
-                enableParquetMemoryOptimization,
-                maxChunkSizeInBytes,
-                maxRowSizeInBytes);
+                clientBufferParameters);
       default:
         throw new SFException(
             ErrorCode.INTERNAL_ERROR, "Unsupported BDEC format version: " + bdecVersion);
@@ -562,10 +559,10 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
   }
 
   private void checkBatchSizeEnforcedMaximum(float batchSizeInBytes) {
-    if (batchSizeInBytes > maxChunkSizeInBytes) {
+    if (batchSizeInBytes > clientBufferParameters.getMaxChunkSizeInBytes()) {
       throw new SFException(
           ErrorCode.MAX_BATCH_SIZE_EXCEEDED,
-          maxChunkSizeInBytes,
+          clientBufferParameters.getMaxChunkSizeInBytes(),
           INSERT_ROWS_RECOMMENDED_MAX_BATCH_SIZE_IN_BYTES);
     }
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ClientBufferParameters.java b/src/main/java/net/snowflake/ingest/streaming/internal/ClientBufferParameters.java
new file mode 100644
index 000000000..dffd824c8
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ClientBufferParameters.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import net.snowflake.ingest.utils.ParameterProvider;
+
+/** Channel's buffer relevant parameters that are set at the owning client level. */
+public class ClientBufferParameters {
+
+  private boolean enableParquetInternalBuffering;
+
+  private long maxChunkSizeInBytes;
+
+  private long maxAllowedRowSizeInBytes;
+
+  /**
+   * Private constructor used for test methods
+   *
+   * @param enableParquetInternalBuffering flag whether buffering in internal Parquet buffers is
+   *     enabled
+   * @param maxChunkSizeInBytes maximum chunk size in bytes
+   * @param maxAllowedRowSizeInBytes maximum row size in bytes
+   */
+  private ClientBufferParameters(
+      boolean enableParquetInternalBuffering,
+      long maxChunkSizeInBytes,
+      long maxAllowedRowSizeInBytes) {
+    this.enableParquetInternalBuffering = enableParquetInternalBuffering;
+    this.maxChunkSizeInBytes = maxChunkSizeInBytes;
+    this.maxAllowedRowSizeInBytes = maxAllowedRowSizeInBytes;
+  }
+
+  /** @param clientInternal reference to the client object where the relevant parameters are set */
+  public ClientBufferParameters(SnowflakeStreamingIngestClientInternal clientInternal) {
+    this.enableParquetInternalBuffering =
+        clientInternal != null
+            ? clientInternal.getParameterProvider().getEnableParquetInternalBuffering()
+            : ParameterProvider.ENABLE_PARQUET_INTERNAL_BUFFERING_DEFAULT;
+    this.maxChunkSizeInBytes =
+        clientInternal != null
+            ? clientInternal.getParameterProvider().getMaxChunkSizeInBytes()
+            : ParameterProvider.MAX_CHUNK_SIZE_IN_BYTES_DEFAULT;
+    this.maxAllowedRowSizeInBytes =
+        clientInternal != null
+            ? clientInternal.getParameterProvider().getMaxAllowedRowSizeInBytes()
+            : ParameterProvider.MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT;
+  }
+
+  /**
+   * @param enableParquetInternalBuffering flag whether buffering in internal Parquet buffers is
+   *     enabled
+   * @param maxChunkSizeInBytes maximum chunk size in bytes
+   * @param maxAllowedRowSizeInBytes maximum row size in bytes
+   * @return ClientBufferParameters object
+   */
+  public static ClientBufferParameters test_createClientBufferParameters(
+      boolean enableParquetInternalBuffering,
+      long maxChunkSizeInBytes,
+      long maxAllowedRowSizeInBytes) {
+    return new ClientBufferParameters(
+        enableParquetInternalBuffering, maxChunkSizeInBytes, maxAllowedRowSizeInBytes);
+  }
+
+  public boolean getEnableParquetInternalBuffering() {
+    return enableParquetInternalBuffering;
+  }
+
+  public long getMaxChunkSizeInBytes() {
+    return maxChunkSizeInBytes;
+  }
+
+  public long getMaxAllowedRowSizeInBytes() {
+    return maxAllowedRowSizeInBytes;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 94baaf384..289b8a983 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -51,9 +51,6 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
   private final String channelName;
 
   private MessageType schema;
-  private final boolean enableParquetInternalBuffering;
-  private final long maxChunkSizeInBytes;
-  private final long maxAllowedRowSizeInBytes;
 
   /** Construct a ParquetRowBuffer object. */
   ParquetRowBuffer(
@@ -62,24 +59,19 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelRuntimeState,
-      boolean enableParquetInternalBuffering,
-      long maxChunkSizeInBytes,
-      long maxAllowedRowSizeInBytes) {
+      ClientBufferParameters clientBufferParameters) {
     super(
         onErrorOption,
         defaultTimezone,
         fullyQualifiedChannelName,
         rowSizeMetric,
         channelRuntimeState,
-        maxChunkSizeInBytes);
+        clientBufferParameters);
     this.fieldIndex = new HashMap<>();
     this.metadata = new HashMap<>();
     this.data = new ArrayList<>();
     this.tempData = new ArrayList<>();
     this.channelName = fullyQualifiedChannelName;
-    this.enableParquetInternalBuffering = enableParquetInternalBuffering;
-    this.maxChunkSizeInBytes = maxChunkSizeInBytes;
-    this.maxAllowedRowSizeInBytes = maxAllowedRowSizeInBytes;
   }
 
   @Override
@@ -122,9 +114,14 @@ public void setupSchema(List<ColumnMetadata> columns) {
   private void createFileWriter() {
     fileOutput = new ByteArrayOutputStream();
     try {
-      if (enableParquetInternalBuffering) {
+      if (clientBufferParameters.getEnableParquetInternalBuffering()) {
         bdecParquetWriter =
-            new BdecParquetWriter(fileOutput, schema, metadata, channelName, maxChunkSizeInBytes);
+            new BdecParquetWriter(
+                fileOutput,
+                schema,
+                metadata,
+                channelName,
+                clientBufferParameters.getMaxChunkSizeInBytes());
       } else {
         this.bdecParquetWriter = null;
       }
@@ -150,7 +147,7 @@ float addRow(
   }
 
   void writeRow(List<Object> row) {
-    if (enableParquetInternalBuffering) {
+    if (clientBufferParameters.getEnableParquetInternalBuffering()) {
       bdecParquetWriter.writeRow(row);
     } else {
       data.add(row);
@@ -209,11 +206,12 @@ private float addRow(
 
     long rowSizeRoundedUp = Double.valueOf(Math.ceil(size)).longValue();
 
-    if (rowSizeRoundedUp > maxAllowedRowSizeInBytes) {
+    if (rowSizeRoundedUp > clientBufferParameters.getMaxAllowedRowSizeInBytes()) {
       throw new SFException(
           ErrorCode.MAX_ROW_SIZE_EXCEEDED,
           String.format(
-              "rowSizeInBytes=%.3f maxAllowedRowSizeInBytes=%d", size, maxAllowedRowSizeInBytes));
+              "rowSizeInBytes=%.3f maxAllowedRowSizeInBytes=%d",
+              size, clientBufferParameters.getMaxAllowedRowSizeInBytes()));
     }
 
     out.accept(Arrays.asList(indexedRow));
@@ -257,7 +255,7 @@ Optional<ParquetChunkData> getSnapshot(final String filePath) {
     metadata.put(Constants.PRIMARY_FILE_ID_KEY, StreamingIngestUtils.getShortname(filePath));
 
     List<List<Object>> oldData = new ArrayList<>();
-    if (!enableParquetInternalBuffering) {
+    if (!clientBufferParameters.getEnableParquetInternalBuffering()) {
       data.forEach(r -> oldData.add(new ArrayList<>(r)));
     }
     return bufferedRowCount <= 0
@@ -321,7 +319,10 @@ void closeInternal() {
 
   @Override
   public Flusher<ParquetChunkData> createFlusher() {
-    return new ParquetFlusher(schema, enableParquetInternalBuffering, maxChunkSizeInBytes);
+    return new ParquetFlusher(
+        schema,
+        clientBufferParameters.getEnableParquetInternalBuffering(),
+        clientBufferParameters.getMaxChunkSizeInBytes());
   }
 
   private static class ParquetColumn {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 07f64cc58..4a27aba5d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -24,7 +24,6 @@
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
-import net.snowflake.ingest.utils.ParameterProvider;
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.Utils;
 
@@ -122,15 +121,7 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
             getFullyQualifiedName(),
             this::collectRowSize,
             channelState,
-            owningClient != null
-                ? owningClient.getParameterProvider().getEnableParquetInternalBuffering()
-                : ParameterProvider.ENABLE_PARQUET_INTERNAL_BUFFERING_DEFAULT,
-            owningClient != null
-                ? owningClient.getParameterProvider().getMaxChunkSizeInBytes()
-                : ParameterProvider.MAX_CHUNK_SIZE_IN_BYTES_DEFAULT,
-            owningClient != null
-                ? owningClient.getParameterProvider().getMaxAllowedRowSizeInBytes()
-                : ParameterProvider.MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT);
+            new ClientBufferParameters(owningClient));
     logger.logInfo(
         "Channel={} created for table={}",
         this.channelFlushContext.getName(),
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 49cf3dce8..63340e25a 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -115,9 +115,10 @@ private AbstractRowBuffer<?> createTestBuffer(OpenChannelRequest.OnErrorOption o
         "test.buffer",
         rs -> {},
         initialState,
-        enableParquetMemoryOptimization,
-        MAX_CHUNK_SIZE_IN_BYTES_DEFAULT,
-        MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT);
+        ClientBufferParameters.test_createClientBufferParameters(
+            enableParquetMemoryOptimization,
+            MAX_CHUNK_SIZE_IN_BYTES_DEFAULT,
+            MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT));
   }
 
   @Test

From b92e3ce5c0a222581acb1b62bbb63a0e07b46b32 Mon Sep 17 00:00:00 2001
From: Tyler Jones <59716821+sfc-gh-tjones@users.noreply.github.com>
Date: Wed, 13 Sep 2023 19:54:17 -0700
Subject: [PATCH 286/356] SNOW-914666 Adds MAX_CLIENT_LAG configuration option
 (#586)

SNOW-914666 Adds MAX_CLIENT_LAG configuration option

We want to expose a knob that gives users the ability to control when
data is ingested. This has a material difference on the size of blobs
generated and can result in fewer smaller-sized blobs which in turn
affects query performance. The trade-off is higher ingest latencies. We
have decided to expose this in the form of an optional `MAX_CLIENT_LAG`
option that accepts inputs as the following:

- `number second` (ex: `1 second`)
- `number seconds` (ex: `2 seconds`)
- `number minute` (ex: `1 minute`)
- `number minutes` (ex: `2 minutes`)

By default we use 1 second as the maximum client lag which is the
current behavior of the SDK.

Note that this dictates when a flush is triggered to cloud storage.
Depending on your connection to cloud storage and cloud storage tail
latencies a blob persist may take longer than expected. Therefore, it is
helpful to think of this parameter as a target, rather than an absolute
number.

If you have manually overridden any flush-related parameters in the past
this can be disabled by setting `MAX_CLIENT_LAG_ENABLED` to `false`
in your configuration file.

@test Adds tests to `ParameterProviderTest`
---
 .../ingest/utils/ParameterProvider.java       |  92 ++++++++++-
 .../internal/ParameterProviderTest.java       | 155 +++++++++++++++++-
 2 files changed, 238 insertions(+), 9 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
index 7e4c3d042..5c6f81f66 100644
--- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
+++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -3,6 +3,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Properties;
+import java.util.concurrent.TimeUnit;
 
 /** Utility class to provide configurable constants */
 public class ParameterProvider {
@@ -31,6 +32,10 @@ public class ParameterProvider {
   public static final String MAX_ALLOWED_ROW_SIZE_IN_BYTES =
       "MAX_ALLOWED_ROW_SIZE_IN_BYTES".toLowerCase();
 
+  public static final String MAX_CLIENT_LAG = "MAX_CLIENT_LAG".toLowerCase();
+
+  public static final String MAX_CLIENT_LAG_ENABLED = "MAX_CLIENT_LAG_ENABLED".toLowerCase();
+
   // Default values
   public static final long BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT = 1000;
   public static final long BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_DEFAULT = 100;
@@ -45,6 +50,14 @@ public class ParameterProvider {
   public static final long MAX_MEMORY_LIMIT_IN_BYTES_DEFAULT = -1L;
   public static final long MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT = 32000000L;
   public static final long MAX_CHUNK_SIZE_IN_BYTES_DEFAULT = 128000000L;
+
+  // Lag related parameters
+  public static final String MAX_CLIENT_LAG_DEFAULT = "1 second";
+  public static final boolean MAX_CLIENT_LAG_ENABLED_DEFAULT = true;
+
+  static final long MAX_CLIENT_LAG_MS_MIN = TimeUnit.SECONDS.toMillis(1);
+
+  static final long MAX_CLIENT_LAG_MS_MAX = TimeUnit.MINUTES.toMillis(10);
   public static final long MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT = 64 * 1024 * 1024; // 64 MB
 
   /* Parameter that enables using internal Parquet buffers for buffering of rows before serializing.
@@ -54,6 +67,9 @@ public class ParameterProvider {
   /** Map of parameter name to parameter value. This will be set by client/configure API Call. */
   private final Map<String, Object> parameterMap = new HashMap<>();
 
+  // Cached buffer flush interval - avoid parsing each time for quick lookup
+  private Long cachedBufferFlushIntervalMs = -1L;
+
   /**
    * Constructor. Takes properties from profile file and properties from client constructor and
    * resolves final parameter value
@@ -81,6 +97,7 @@ private void updateValue(
       this.parameterMap.put(key, props.getOrDefault(key, defaultValue));
     }
   }
+
   /**
    * Sets parameter values by first checking 1. parameterOverrides 2. props 3. default value
    *
@@ -149,17 +166,80 @@ private void setParameterMap(Map<String, Object> parameterOverrides, Properties
 
     this.updateValue(
         MAX_CHUNK_SIZE_IN_BYTES, MAX_CHUNK_SIZE_IN_BYTES_DEFAULT, parameterOverrides, props);
+
+    this.updateValue(MAX_CLIENT_LAG, MAX_CLIENT_LAG_DEFAULT, parameterOverrides, props);
+    this.updateValue(
+        MAX_CLIENT_LAG_ENABLED, MAX_CLIENT_LAG_ENABLED_DEFAULT, parameterOverrides, props);
   }
 
   /** @return Longest interval in milliseconds between buffer flushes */
   public long getBufferFlushIntervalInMs() {
-    Object val =
-        this.parameterMap.getOrDefault(
-            BUFFER_FLUSH_INTERVAL_IN_MILLIS, BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT);
-    if (val instanceof String) {
-      return Long.parseLong(val.toString());
+    if (getMaxClientLagEnabled()) {
+      if (cachedBufferFlushIntervalMs != -1L) {
+        return cachedBufferFlushIntervalMs;
+      }
+      long lag = getMaxClientLagMs();
+      if (cachedBufferFlushIntervalMs == -1L) {
+        cachedBufferFlushIntervalMs = lag;
+      }
+      return cachedBufferFlushIntervalMs;
+    } else {
+      Object val =
+          this.parameterMap.getOrDefault(
+              BUFFER_FLUSH_INTERVAL_IN_MILLIS, BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT);
+      return (val instanceof String) ? Long.parseLong(val.toString()) : (long) val;
     }
-    return (long) val;
+  }
+
+  private long getMaxClientLagMs() {
+    Object val = this.parameterMap.getOrDefault(MAX_CLIENT_LAG, MAX_CLIENT_LAG_DEFAULT);
+    if (!(val instanceof String)) {
+      return BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT;
+    }
+    String maxLag = (String) val;
+    String[] lagParts = maxLag.split(" ");
+    if (lagParts.length != 2
+        || (lagParts[0] == null || "".equals(lagParts[0]))
+        || (lagParts[1] == null || "".equals(lagParts[1]))) {
+      throw new IllegalArgumentException(
+          String.format("Failed to parse MAX_CLIENT_LAG = '%s'", maxLag));
+    }
+    long lag;
+    try {
+      lag = Long.parseLong(lagParts[0]);
+    } catch (Throwable t) {
+      throw new IllegalArgumentException(
+          String.format("Failed to parse MAX_CLIENT_LAG = '%s'", lagParts[0]), t);
+    }
+    long computedLag;
+    switch (lagParts[1].toLowerCase()) {
+      case "second":
+      case "seconds":
+        computedLag = lag * TimeUnit.SECONDS.toMillis(1);
+        break;
+      case "minute":
+      case "minutes":
+        computedLag = lag * TimeUnit.SECONDS.toMillis(60);
+        break;
+      default:
+        throw new IllegalArgumentException(
+            String.format("Invalid time unit supplied = '%s", lagParts[1]));
+    }
+
+    if (!(computedLag >= MAX_CLIENT_LAG_MS_MIN && computedLag <= MAX_CLIENT_LAG_MS_MAX)) {
+      throw new IllegalArgumentException(
+          String.format(
+              "Lag falls outside of allowed time range. Minimum (milliseconds) = %s, Maximum"
+                  + " (milliseconds) = %s",
+              MAX_CLIENT_LAG_MS_MIN, MAX_CLIENT_LAG_MS_MAX));
+    }
+    return computedLag;
+  }
+
+  private boolean getMaxClientLagEnabled() {
+    Object val =
+        this.parameterMap.getOrDefault(MAX_CLIENT_LAG_ENABLED, MAX_CLIENT_LAG_ENABLED_DEFAULT);
+    return (val instanceof String) ? Boolean.parseBoolean(val.toString()) : (boolean) val;
   }
 
   /** @return Time in milliseconds between checks to see if the buffer should be flushed */
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
index dcf4037c6..def5f7ecf 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
@@ -9,9 +9,7 @@
 
 public class ParameterProviderTest {
 
-  @Test
-  public void withValuesSet() {
-    Properties prop = new Properties();
+  private Map<String, Object> getStartingParameterMap() {
     Map<String, Object> parameterMap = new HashMap<>();
     parameterMap.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS, 3L);
     parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 4L);
@@ -22,6 +20,14 @@ public void withValuesSet() {
     parameterMap.put(ParameterProvider.BLOB_UPLOAD_MAX_RETRY_COUNT, 100);
     parameterMap.put(ParameterProvider.MAX_MEMORY_LIMIT_IN_BYTES, 1000L);
     parameterMap.put(ParameterProvider.MAX_CHANNEL_SIZE_IN_BYTES, 1000000L);
+    return parameterMap;
+  }
+
+  @Test
+  public void withValuesSet() {
+    Properties prop = new Properties();
+    Map<String, Object> parameterMap = getStartingParameterMap();
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, false);
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
 
     Assert.assertEquals(3L, parameterProvider.getBufferFlushIntervalInMs());
@@ -42,6 +48,7 @@ public void withNullProps() {
     parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 4L);
     parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE, 6);
     parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_BYTES, 1024);
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, false);
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, null);
 
     Assert.assertEquals(3, parameterProvider.getBufferFlushIntervalInMs());
@@ -60,6 +67,7 @@ public void withNullParameterMap() {
     props.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 4L);
     props.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE, 6);
     props.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_BYTES, 1024);
+    props.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, false);
     ParameterProvider parameterProvider = new ParameterProvider(null, props);
 
     Assert.assertEquals(3, parameterProvider.getBufferFlushIntervalInMs());
@@ -123,4 +131,145 @@ public void withDefaultValues() {
         ParameterProvider.MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT,
         parameterProvider.getMaxChannelSizeInBytes());
   }
+
+  @Test
+  public void testMaxClientLagEnabled() {
+    Properties prop = new Properties();
+    Map<String, Object> parameterMap = getStartingParameterMap();
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "2 second");
+    ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+    Assert.assertEquals(2000, parameterProvider.getBufferFlushIntervalInMs());
+    // call again to trigger caching logic
+    Assert.assertEquals(2000, parameterProvider.getBufferFlushIntervalInMs());
+  }
+
+  @Test
+  public void testMaxClientLagEnabledPluralTimeUnit() {
+    Properties prop = new Properties();
+    Map<String, Object> parameterMap = getStartingParameterMap();
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "2 seconds");
+    ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+    Assert.assertEquals(2000, parameterProvider.getBufferFlushIntervalInMs());
+  }
+
+  @Test
+  public void testMaxClientLagEnabledMinuteTimeUnit() {
+    Properties prop = new Properties();
+    Map<String, Object> parameterMap = getStartingParameterMap();
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "1 minute");
+    ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+    Assert.assertEquals(60000, parameterProvider.getBufferFlushIntervalInMs());
+  }
+
+  @Test
+  public void testMaxClientLagEnabledMinuteTimeUnitPluralTimeUnit() {
+    Properties prop = new Properties();
+    Map<String, Object> parameterMap = getStartingParameterMap();
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "2 minutes");
+    ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+    Assert.assertEquals(120000, parameterProvider.getBufferFlushIntervalInMs());
+  }
+
+  @Test
+  public void testMaxClientLagEnabledDefaultValue() {
+    Properties prop = new Properties();
+    Map<String, Object> parameterMap = getStartingParameterMap();
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
+    ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+    Assert.assertEquals(1000, parameterProvider.getBufferFlushIntervalInMs());
+  }
+
+  @Test
+  public void testMaxClientLagEnabledMissingUnit() {
+    Properties prop = new Properties();
+    Map<String, Object> parameterMap = getStartingParameterMap();
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "1");
+    ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+    try {
+      parameterProvider.getBufferFlushIntervalInMs();
+      Assert.fail("Should not have succeeded");
+    } catch (IllegalArgumentException e) {
+      Assert.assertTrue(e.getMessage().startsWith("Failed to parse"));
+    }
+  }
+
+  @Test
+  public void testMaxClientLagEnabledMissingUnitTimeUnitSupplied() {
+    Properties prop = new Properties();
+    Map<String, Object> parameterMap = getStartingParameterMap();
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, " year");
+    ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+    try {
+      parameterProvider.getBufferFlushIntervalInMs();
+      Assert.fail("Should not have succeeded");
+    } catch (IllegalArgumentException e) {
+      Assert.assertTrue(e.getMessage().startsWith("Failed to parse"));
+    }
+  }
+
+  @Test
+  public void testMaxClientLagEnabledInvalidTimeUnit() {
+    Properties prop = new Properties();
+    Map<String, Object> parameterMap = getStartingParameterMap();
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "1 year");
+    ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+    try {
+      parameterProvider.getBufferFlushIntervalInMs();
+      Assert.fail("Should not have succeeded");
+    } catch (IllegalArgumentException e) {
+      Assert.assertTrue(e.getMessage().startsWith("Invalid time unit"));
+    }
+  }
+
+  @Test
+  public void testMaxClientLagEnabledInvalidUnit() {
+    Properties prop = new Properties();
+    Map<String, Object> parameterMap = getStartingParameterMap();
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "banana minute");
+    ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+    try {
+      parameterProvider.getBufferFlushIntervalInMs();
+      Assert.fail("Should not have succeeded");
+    } catch (IllegalArgumentException e) {
+      Assert.assertTrue(e.getMessage().startsWith("Failed to parse"));
+    }
+  }
+
+  @Test
+  public void testMaxClientLagEnabledThresholdBelow() {
+    Properties prop = new Properties();
+    Map<String, Object> parameterMap = getStartingParameterMap();
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "0 second");
+    ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+    try {
+      parameterProvider.getBufferFlushIntervalInMs();
+      Assert.fail("Should not have succeeded");
+    } catch (IllegalArgumentException e) {
+      Assert.assertTrue(e.getMessage().startsWith("Lag falls outside"));
+    }
+  }
+
+  @Test
+  public void testMaxClientLagEnabledThresholdAbove() {
+    Properties prop = new Properties();
+    Map<String, Object> parameterMap = getStartingParameterMap();
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "11 minutes");
+    ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+    try {
+      parameterProvider.getBufferFlushIntervalInMs();
+      Assert.fail("Should not have succeeded");
+    } catch (IllegalArgumentException e) {
+      Assert.assertTrue(e.getMessage().startsWith("Lag falls outside"));
+    }
+  }
 }

From 8318bd7c968c81060291a5c04096647f6f76b057 Mon Sep 17 00:00:00 2001
From: Tyler Jones <59716821+sfc-gh-tjones@users.noreply.github.com>
Date: Thu, 21 Sep 2023 11:03:51 -0700
Subject: [PATCH 287/356] SNOW-919423 Surfaces Table Schema for a Channel
 (#589)

We've had requests to surface the table schema for a channel so that it
is possible to reason about what columns need to be supplied to a given
call of `insertRow` or `insertRows`. This surfaces a map of Column
Name to Column Properties that are normally surfaced in the output of
`SHOW COLUMNS`.

@test adds test to `SnowflakeStreamingIngestChannelTest.java`
---
 .../SnowflakeStreamingIngestChannel.java      | 10 ++++
 .../streaming/internal/ColumnProperties.java  | 60 +++++++++++++++++++
 ...owflakeStreamingIngestChannelInternal.java | 12 ++++
 .../SnowflakeStreamingIngestChannelTest.java  | 16 +++++
 4 files changed, 98 insertions(+)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/ColumnProperties.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
index 5fdadc90b..909fcefa1 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
@@ -8,6 +8,7 @@
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import javax.annotation.Nullable;
+import net.snowflake.ingest.streaming.internal.ColumnProperties;
 
 /**
  * A logical partition that represents a connection to a single Snowflake table, data will be
@@ -253,4 +254,13 @@ InsertValidationResponse insertRows(
    */
   @Nullable
   String getLatestCommittedOffsetToken();
+
+  /**
+   * Gets the table schema associated with this channel. Note that this is the table schema at the
+   * time of a channel open event. The schema may be changed on the Snowflake side in which case
+   * this will continue to show an old schema version until the channel is re-opened.
+   *
+   * @return map representing Column Name -> Column Properties
+   */
+  Map<String, ColumnProperties> getTableSchema();
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ColumnProperties.java b/src/main/java/net/snowflake/ingest/streaming/internal/ColumnProperties.java
new file mode 100644
index 000000000..856170c57
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ColumnProperties.java
@@ -0,0 +1,60 @@
+package net.snowflake.ingest.streaming.internal;
+
+/**
+ * Class that encapsulates column properties. These are the same properties showed in the output of
+ * <a href="https://docs.snowflake.com/en/sql-reference/sql/show-columns">SHOW COLUMNS</a>. Note
+ * that this is slightly different than the internal column metadata used elsewhere in this SDK.
+ */
+public class ColumnProperties {
+  private String type;
+
+  private String logicalType;
+
+  private Integer precision;
+
+  private Integer scale;
+
+  private Integer byteLength;
+
+  private Integer length;
+
+  private boolean nullable;
+
+  ColumnProperties(ColumnMetadata columnMetadata) {
+    this.type = columnMetadata.getType();
+    this.logicalType = columnMetadata.getLogicalType();
+    this.precision = columnMetadata.getPrecision();
+    this.scale = columnMetadata.getScale();
+    this.byteLength = columnMetadata.getByteLength();
+    this.length = columnMetadata.getLength();
+    this.nullable = columnMetadata.getNullable();
+  }
+
+  public String getType() {
+    return type;
+  }
+
+  public String getLogicalType() {
+    return logicalType;
+  }
+
+  public Integer getPrecision() {
+    return precision;
+  }
+
+  public Integer getScale() {
+    return scale;
+  }
+
+  public Integer getByteLength() {
+    return byteLength;
+  }
+
+  public Integer getLength() {
+    return length;
+  }
+
+  public boolean isNullable() {
+    return nullable;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 4a27aba5d..756f9b5e8 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -12,6 +12,7 @@
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -51,6 +52,9 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
   // State of the channel that will be shared with its underlying buffer
   private final ChannelRuntimeState channelState;
 
+  // Internal map of column name -> column properties
+  private final Map<String, ColumnProperties> tableColumns;
+
   /**
    * Constructor for TESTING ONLY which allows us to set the test mode
    *
@@ -122,6 +126,7 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
             this::collectRowSize,
             channelState,
             new ClientBufferParameters(owningClient));
+    this.tableColumns = new HashMap<>();
     logger.logInfo(
         "Channel={} created for table={}",
         this.channelFlushContext.getName(),
@@ -298,6 +303,7 @@ public CompletableFuture<Void> close() {
   void setupSchema(List<ColumnMetadata> columns) {
     logger.logDebug("Setup schema for channel={}, schema={}", getFullyQualifiedName(), columns);
     this.rowBuffer.setupSchema(columns);
+    columns.forEach(c -> tableColumns.putIfAbsent(c.getName(), new ColumnProperties(c)));
   }
 
   /**
@@ -391,6 +397,12 @@ public String getLatestCommittedOffsetToken() {
     return response.getPersistedOffsetToken();
   }
 
+  /** Returns a map of column name -> datatype for the table the channel is bound to */
+  @Override
+  public Map<String, ColumnProperties> getTableSchema() {
+    return this.tableColumns;
+  }
+
   /** Check whether we need to throttle the insertRows API */
   void throttleInsertIfNeeded(MemoryInfoProvider memoryInfoProvider) {
     int retry = 0;
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 62f2efdce..e9dc87e94 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -495,6 +495,22 @@ public void testOpenChannelSuccessResponse() throws Exception {
     Assert.assertEquals(dbName, channel.getDBName());
     Assert.assertEquals(schemaName, channel.getSchemaName());
     Assert.assertEquals(tableName, channel.getTableName());
+
+    Assert.assertTrue(channel.getTableSchema().containsKey("C1"));
+    Assert.assertEquals("NUMBER(38,0)", channel.getTableSchema().get("C1").getType());
+    Assert.assertEquals("FIXED", channel.getTableSchema().get("C1").getLogicalType());
+    Assert.assertEquals(38, (int) channel.getTableSchema().get("C1").getPrecision());
+    Assert.assertEquals(0, (int) channel.getTableSchema().get("C1").getScale());
+    Assert.assertNull(channel.getTableSchema().get("C1").getByteLength());
+    Assert.assertTrue(channel.getTableSchema().get("C1").isNullable());
+
+    Assert.assertTrue(channel.getTableSchema().containsKey("C2"));
+    Assert.assertEquals("NUMBER(38,0)", channel.getTableSchema().get("C2").getType());
+    Assert.assertEquals("FIXED", channel.getTableSchema().get("C2").getLogicalType());
+    Assert.assertEquals(38, (int) channel.getTableSchema().get("C2").getPrecision());
+    Assert.assertEquals(0, (int) channel.getTableSchema().get("C2").getScale());
+    Assert.assertNull(channel.getTableSchema().get("C2").getByteLength());
+    Assert.assertTrue(channel.getTableSchema().get("C2").isNullable());
   }
 
   @Test

From 51e99bc58e2f59258ea8c55192ad2059c2b759f6 Mon Sep 17 00:00:00 2001
From: Lorenz Thiede <lorenz.thiede@snowflake.com>
Date: Fri, 22 Sep 2023 11:31:35 +0200
Subject: [PATCH 288/356] NO-SNOW Remove wrong documentation (#591)

NO-SNOW Fix documentation

The documentation of the interface SnowflakeStreamingIngestChannel wrongly said that there was no limit on the number of channels. I removed that sentence.
---
 .../ingest/streaming/SnowflakeStreamingIngestChannel.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
index 909fcefa1..a8bb5db16 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
@@ -18,7 +18,7 @@
  * at the same time. When a new channel is opened, all previously opened channels with the same name
  * are invalidated (this applies for the table globally. not just in a single JVM). In order to
  * ingest data from multiple threads/clients/applications, we recommend opening multiple channels,
- * each with a different name. There is no limit on the number of channels that can be opened.
+ * each with a different name.
  *
  * <p>Thread safety note: Implementations of this interface are required to be thread safe.
  */

From 7b2c6681da6e0b03e73428d3dfdb21b858738610 Mon Sep 17 00:00:00 2001
From: Dhara Shah <dhara.shah@snowflake.com>
Date: Fri, 22 Sep 2023 11:47:16 -0700
Subject: [PATCH 289/356] updated SDK version to 2.0.4-SNAPSHOT (#588)

---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 360e70c38..45b739a18 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.0.3</version>
+  <version>2.0.4-SNAPSHOT</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index a8018954c..6936bc52a 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.0.3";
+  public static final String DEFAULT_VERSION = "2.0.4-SNAPSHOT";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 46e06dd1bd416a93247d2d8ac49548b646f437eb Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Wed, 27 Sep 2023 17:07:56 +0200
Subject: [PATCH 290/356] SNOW-924864 Upgrade snappy-java (#592)

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 45b739a18..dbf156690 100644
--- a/pom.xml
+++ b/pom.xml
@@ -63,7 +63,7 @@
     <protobuf.version>3.19.6</protobuf.version>
     <shadeBase>net.snowflake.ingest.internal</shadeBase>
     <slf4j.version>1.7.36</slf4j.version>
-    <snappy.version>1.1.10.1</snappy.version>
+    <snappy.version>1.1.10.4</snappy.version>
     <snowjdbc.version>3.13.30</snowjdbc.version>
     <yetus.version>0.13.0</yetus.version>
   </properties>

From 955ff640e0aa54b0fa63fc59025cf3033d9c0d7f Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 27 Sep 2023 18:38:59 -0700
Subject: [PATCH 291/356] SNOW-918949: Add row index information to all
 exceptions (#590)

Add row index information to all exceptions, also unify the format so it can be easily parsed
---
 .../streaming/internal/AbstractRowBuffer.java | 15 +++--
 .../internal/DataValidationUtil.java          | 17 ++++--
 .../streaming/internal/ParquetRowBuffer.java  |  4 +-
 .../internal/DataValidationUtilTest.java      | 60 +++++++++----------
 .../streaming/internal/RowBufferTest.java     |  4 +-
 .../SnowflakeStreamingIngestChannelTest.java  |  4 +-
 6 files changed, 56 insertions(+), 48 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index fa2c65006..dde639d00 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -236,10 +236,11 @@ public float getSize() {
    *
    * @param row the input row
    * @param error the insert error that we return to the customer
+   * @param rowIndex the index of the current row in the input batch
    * @return the set of input column names
    */
   Set<String> verifyInputColumns(
-      Map<String, Object> row, InsertValidationResponse.InsertError error) {
+      Map<String, Object> row, InsertValidationResponse.InsertError error, int rowIndex) {
     // Map of unquoted column name -> original column name
     Map<String, String> inputColNamesMap =
         row.keySet().stream()
@@ -260,7 +261,8 @@ Set<String> verifyInputColumns(
       throw new SFException(
           ErrorCode.INVALID_FORMAT_ROW,
           "Extra columns: " + extraCols,
-          "Columns not present in the table shouldn't be specified.");
+          String.format(
+              "Columns not present in the table shouldn't be specified, rowIndex:%d", rowIndex));
     }
 
     // Check for missing columns in the row
@@ -278,7 +280,8 @@ Set<String> verifyInputColumns(
       throw new SFException(
           ErrorCode.INVALID_FORMAT_ROW,
           "Missing columns: " + missingCols,
-          "Values for all non-nullable columns must be specified.");
+          String.format(
+              "Values for all non-nullable columns must be specified, rowIndex:%d", rowIndex));
     }
 
     return inputColNamesMap.keySet();
@@ -304,12 +307,12 @@ public InsertValidationResponse insertRows(
       this.channelState.updateInsertStats(System.currentTimeMillis(), this.bufferedRowCount);
       if (onErrorOption == OpenChannelRequest.OnErrorOption.CONTINUE) {
         // Used to map incoming row(nth row) to InsertError(for nth row) in response
-        long rowIndex = 0;
+        int rowIndex = 0;
         for (Map<String, Object> row : rows) {
           InsertValidationResponse.InsertError error =
               new InsertValidationResponse.InsertError(row, rowIndex);
           try {
-            Set<String> inputColumnNames = verifyInputColumns(row, error);
+            Set<String> inputColumnNames = verifyInputColumns(row, error, rowIndex);
             rowsSizeInBytes +=
                 addRow(row, this.bufferedRowCount, this.statsMap, inputColumnNames, rowIndex);
             this.bufferedRowCount++;
@@ -333,7 +336,7 @@ public InsertValidationResponse insertRows(
         float tempRowsSizeInBytes = 0F;
         int tempRowCount = 0;
         for (Map<String, Object> row : rows) {
-          Set<String> inputColumnNames = verifyInputColumns(row, null);
+          Set<String> inputColumnNames = verifyInputColumns(row, null, tempRowCount);
           tempRowsSizeInBytes +=
               addTempRow(row, tempRowCount, this.tempStatsMap, inputColumnNames, tempRowCount);
           checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index 1bdfc2095..a1831f829 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -457,7 +457,10 @@ static TimestampWrapper validateAndParseTimestamp(
     if (offsetDateTime.getYear() < 1 || offsetDateTime.getYear() > 9999) {
       throw new SFException(
           ErrorCode.INVALID_VALUE_ROW,
-          "Timestamp out of representable inclusive range of years between 1 and 9999");
+          String.format(
+              "Timestamp out of representable inclusive range of years between 1 and 9999,"
+                  + " rowIndex:%d",
+              insertRowIndex));
     }
     return new TimestampWrapper(offsetDateTime, scale);
   }
@@ -588,7 +591,10 @@ static int validateAndParseDate(String columnName, Object input, long insertRowI
     if (offsetDateTime.getYear() < -9999 || offsetDateTime.getYear() > 9999) {
       throw new SFException(
           ErrorCode.INVALID_VALUE_ROW,
-          "Date out of representable inclusive range of years between -9999 and 9999");
+          String.format(
+              "Date out of representable inclusive range of years between -9999 and 9999,"
+                  + " rowIndex:%d",
+              insertRowIndex));
     }
 
     return Math.toIntExact(offsetDateTime.toLocalDate().toEpochDay());
@@ -814,7 +820,7 @@ static void checkValueInRange(
       throw new SFException(
           ErrorCode.INVALID_FORMAT_ROW,
           String.format(
-              "Number out of representable exclusive range of (-1e%s..1e%s), Row Index:%s",
+              "Number out of representable exclusive range of (-1e%s..1e%s), rowIndex:%d",
               precision - scale, precision - scale, insertRowIndex));
     }
   }
@@ -858,8 +864,7 @@ private static SFException typeNotAllowedException(
     return new SFException(
         ErrorCode.INVALID_FORMAT_ROW,
         String.format(
-            "Object of type %s cannot be ingested into Snowflake column %s of type %s, Row"
-                + " Index:%s",
+            "Object of type %s cannot be ingested into Snowflake column %s of type %s, rowIndex:%d",
             javaType.getName(), columnName, snowflakeType, insertRowIndex),
         String.format(
             String.format("Allowed Java types: %s", String.join(", ", allowedJavaTypes))));
@@ -882,7 +887,7 @@ private static SFException valueFormatNotAllowedException(
     return new SFException(
         ErrorCode.INVALID_VALUE_ROW,
         String.format(
-            "Value cannot be ingested into Snowflake column %s of type %s, Row Index: %s, reason:"
+            "Value cannot be ingested into Snowflake column %s of type %s, rowIndex:%d, reason:"
                 + " %s",
             columnName, snowflakeType, rowIndex, reason));
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 289b8a983..75966eb35 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -210,8 +210,8 @@ private float addRow(
       throw new SFException(
           ErrorCode.MAX_ROW_SIZE_EXCEEDED,
           String.format(
-              "rowSizeInBytes=%.3f maxAllowedRowSizeInBytes=%d",
-              size, clientBufferParameters.getMaxAllowedRowSizeInBytes()));
+              "rowSizeInBytes:%.3f, maxAllowedRowSizeInBytes:%d, rowIndex:%d",
+              size, clientBufferParameters.getMaxAllowedRowSizeInBytes(), insertRowsCurrIndex));
     }
 
     out.accept(Arrays.asList(indexedRow));
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index f6f9e36af..86706fcf2 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -676,19 +676,19 @@ public void testTooLargeMultiByteSemiStructuredValues() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type VARIANT, Row Index: 0, reason:"
+            + " cannot be ingested into Snowflake column COL of type VARIANT, rowIndex:0, reason:"
             + " Variant too long: length=18874376 maxLength=16777152",
         () -> validateAndParseVariant("COL", m, 0));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type ARRAY, Row Index: 0, reason:"
+            + " cannot be ingested into Snowflake column COL of type ARRAY, rowIndex:0, reason:"
             + " Array too large. length=18874378 maxLength=16777152",
         () -> validateAndParseArray("COL", m, 0));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type OBJECT, Row Index: 0, reason:"
+            + " cannot be ingested into Snowflake column COL of type OBJECT, rowIndex:0, reason:"
             + " Object too large. length=18874376 maxLength=16777152",
         () -> validateAndParseObject("COL", m, 0));
   }
@@ -1005,13 +1005,13 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type BOOLEAN, Row Index:0. Allowed"
+            + " cannot be ingested into Snowflake column COL of type BOOLEAN, rowIndex:0. Allowed"
             + " Java types: boolean, Number, String",
         () -> validateAndParseBoolean("COL", new Object(), 0));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type BOOLEAN, Row Index: 0, reason:"
+            + " cannot be ingested into Snowflake column COL of type BOOLEAN, rowIndex:0, reason:"
             + " Not a valid boolean, see"
             + " https://docs.snowflake.com/en/sql-reference/data-types-logical.html#conversion-to-boolean"
             + " for the list of supported formats",
@@ -1021,13 +1021,13 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type TIME, Row Index:0. Allowed"
+            + " cannot be ingested into Snowflake column COL of type TIME, rowIndex:0. Allowed"
             + " Java types: String, LocalTime, OffsetTime",
         () -> validateAndParseTime("COL", new Object(), 10, 0));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type TIME, Row Index: 0, reason:"
+            + " cannot be ingested into Snowflake column COL of type TIME, rowIndex:0, reason:"
             + " Not a valid time, see"
             + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
             + " the list of supported formats",
@@ -1037,13 +1037,13 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type DATE, Row Index:0. Allowed"
+            + " cannot be ingested into Snowflake column COL of type DATE, rowIndex:0. Allowed"
             + " Java types: String, LocalDate, LocalDateTime, ZonedDateTime, OffsetDateTime",
         () -> validateAndParseDate("COL", new Object(), 0));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type DATE, Row Index: 0, reason:"
+            + " cannot be ingested into Snowflake column COL of type DATE, rowIndex:0, reason:"
             + " Not a valid value, see"
             + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
             + " the list of supported formats",
@@ -1053,14 +1053,14 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, Row Index:0."
+            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, rowIndex:0."
             + " Allowed Java types: String, LocalDate, LocalDateTime, ZonedDateTime,"
             + " OffsetDateTime",
         () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, true, 0));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, Row Index: 0,"
+            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, rowIndex:0,"
             + " reason: Not a valid value, see"
             + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
             + " the list of supported formats",
@@ -1070,14 +1070,14 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, Row Index:0."
+            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, rowIndex:0."
             + " Allowed Java types: String, LocalDate, LocalDateTime, ZonedDateTime,"
             + " OffsetDateTime",
         () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, false, 0));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, Row Index: 0,"
+            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, rowIndex:0,"
             + " reason: Not a valid value, see"
             + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
             + " the list of supported formats",
@@ -1087,14 +1087,14 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, Row Index:0."
+            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, rowIndex:0."
             + " Allowed Java types: String, LocalDate, LocalDateTime, ZonedDateTime,"
             + " OffsetDateTime",
         () -> validateAndParseTimestamp("COL", new Object(), 3, UTC, false, 0));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, Row Index: 0,"
+            + " cannot be ingested into Snowflake column COL of type TIMESTAMP, rowIndex:0,"
             + " reason: Not a valid value, see"
             + " https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview for"
             + " the list of supported formats",
@@ -1104,13 +1104,13 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type NUMBER, Row Index:0. Allowed"
+            + " cannot be ingested into Snowflake column COL of type NUMBER, rowIndex:0. Allowed"
             + " Java types: int, long, byte, short, float, double, BigDecimal, BigInteger, String",
         () -> validateAndParseBigDecimal("COL", new Object(), 0));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type NUMBER, Row Index: 0, reason:"
+            + " cannot be ingested into Snowflake column COL of type NUMBER, rowIndex:0, reason:"
             + " Not a valid number",
         () -> validateAndParseBigDecimal("COL", "abc", 0));
 
@@ -1118,13 +1118,13 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type REAL, Row Index:0. Allowed"
+            + " cannot be ingested into Snowflake column COL of type REAL, rowIndex:0. Allowed"
             + " Java types: Number, String",
         () -> validateAndParseReal("COL", new Object(), 0));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type REAL, Row Index: 0, reason:"
+            + " cannot be ingested into Snowflake column COL of type REAL, rowIndex:0, reason:"
             + " Not a valid decimal number",
         () -> validateAndParseReal("COL", "abc", 0));
 
@@ -1132,13 +1132,13 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type STRING, Row Index:0. Allowed"
+            + " cannot be ingested into Snowflake column COL of type STRING, rowIndex:0. Allowed"
             + " Java types: String, Number, boolean, char",
         () -> validateAndParseString("COL", new Object(), Optional.empty(), 0));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type STRING, Row Index: 0, reason:"
+            + " cannot be ingested into Snowflake column COL of type STRING, rowIndex:0, reason:"
             + " String too long: length=3 characters maxLength=2 characters",
         () -> validateAndParseString("COL", "abc", Optional.of(2), 0));
 
@@ -1146,19 +1146,19 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type BINARY, Row Index:0. Allowed"
+            + " cannot be ingested into Snowflake column COL of type BINARY, rowIndex:0. Allowed"
             + " Java types: byte[], String",
         () -> validateAndParseBinary("COL", new Object(), Optional.empty(), 0));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type BINARY, Row Index: 0, reason:"
+            + " cannot be ingested into Snowflake column COL of type BINARY, rowIndex:0, reason:"
             + " Binary too long: length=2 maxLength=1",
         () -> validateAndParseBinary("COL", new byte[] {1, 2}, Optional.of(1), 0));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type BINARY, Row Index: 0, reason:"
+            + " cannot be ingested into Snowflake column COL of type BINARY, rowIndex:0, reason:"
             + " Not a valid hex string",
         () -> validateAndParseBinary("COL", "ghi", Optional.empty(), 0));
 
@@ -1166,14 +1166,14 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type VARIANT, Row Index:0. Allowed"
+            + " cannot be ingested into Snowflake column COL of type VARIANT, rowIndex:0. Allowed"
             + " Java types: String, Primitive data types and their arrays, java.time.*, List<T>,"
             + " Map<String, T>, T[]",
         () -> validateAndParseVariant("COL", new Object(), 0));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type VARIANT, Row Index: 0, reason:"
+            + " cannot be ingested into Snowflake column COL of type VARIANT, rowIndex:0, reason:"
             + " Not a valid JSON",
         () -> validateAndParseVariant("COL", "][", 0));
 
@@ -1181,14 +1181,14 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type ARRAY, Row Index:0. Allowed"
+            + " cannot be ingested into Snowflake column COL of type ARRAY, rowIndex:0. Allowed"
             + " Java types: String, Primitive data types and their arrays, java.time.*, List<T>,"
             + " Map<String, T>, T[]",
         () -> validateAndParseArray("COL", new Object(), 0));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type ARRAY, Row Index: 0, reason:"
+            + " cannot be ingested into Snowflake column COL of type ARRAY, rowIndex:0, reason:"
             + " Not a valid JSON",
         () -> validateAndParseArray("COL", "][", 0));
 
@@ -1196,14 +1196,14 @@ public void testExceptionMessages() {
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_FORMAT_ROW,
         "The given row cannot be converted to the internal format: Object of type java.lang.Object"
-            + " cannot be ingested into Snowflake column COL of type OBJECT, Row Index:0. Allowed"
+            + " cannot be ingested into Snowflake column COL of type OBJECT, rowIndex:0. Allowed"
             + " Java types: String, Primitive data types and their arrays, java.time.*, List<T>,"
             + " Map<String, T>, T[]",
         () -> validateAndParseObject("COL", new Object(), 0));
     expectErrorCodeAndMessage(
         ErrorCode.INVALID_VALUE_ROW,
         "The given row cannot be converted to the internal format due to invalid value: Value"
-            + " cannot be ingested into Snowflake column COL of type OBJECT, Row Index: 0, reason:"
+            + " cannot be ingested into Snowflake column COL of type OBJECT, rowIndex:0, reason:"
             + " Not a valid JSON",
         () -> validateAndParseObject("COL", "}{", 0));
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 63340e25a..8d71d9a44 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -307,8 +307,8 @@ public void testRowIndexWithMultipleRowsWithError() {
             .getMessage()
             .equalsIgnoreCase(
                 "The given row cannot be converted to the internal format due to invalid value:"
-                    + " Value cannot be ingested into Snowflake column COLCHAR of type STRING, Row"
-                    + " Index: 1, reason: String too long: length=22 characters maxLength=11"
+                    + " Value cannot be ingested into Snowflake column COLCHAR of type STRING,"
+                    + " rowIndex:1, reason: String too long: length=22 characters maxLength=11"
                     + " characters"));
   }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index e9dc87e94..8fbf67264 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -588,8 +588,8 @@ public void testInsertTooLargeRow() {
             .collect(Collectors.toList());
 
     String expectedMessage =
-        "The given row exceeds the maximum allowed row size rowSizeInBytes=67109128.000"
-            + " maxAllowedRowSizeInBytes=67108864";
+        "The given row exceeds the maximum allowed row size rowSizeInBytes:67109128.000,"
+            + " maxAllowedRowSizeInBytes:67108864, rowIndex:0";
 
     Map<String, Object> row = new HashMap<>();
     schema.forEach(x -> row.put(x.getName(), byteArrayOneMb));

From 640a56c054d558e7651efd1f849a35e6dfba6642 Mon Sep 17 00:00:00 2001
From: Jay Patel <jay.patel@snowflake.com>
Date: Tue, 3 Oct 2023 11:15:08 -0700
Subject: [PATCH 292/356] =?UTF-8?q?SNOW-928802=20Use=20concurrent=20hashma?=
 =?UTF-8?q?p=20to=20prevent=20ConcurrentModificationE=E2=80=A6=20(#594)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* SNOW-928802 Use concurrent hashmap to prevent ConcurrentModificationException

- We did make a change to use ConcurrentHashMap but that was in test
  code, we still used HashMap! This will hopefully not result into
  ConcurrentModificationException

* Hello Google jfmt
---
 .../net/snowflake/ingest/streaming/internal/FlushService.java  | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 8016f9b5c..ff879b8d6 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -21,7 +21,6 @@
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -174,7 +173,7 @@ List<List<ChannelData<T>>> getData() {
     this.isNeedFlush = false;
     this.lastFlushTime = System.currentTimeMillis();
     this.isTestMode = isTestMode;
-    this.latencyTimerContextMap = new HashMap<>();
+    this.latencyTimerContextMap = new ConcurrentHashMap<>();
     this.bdecVersion = this.owningClient.getParameterProvider().getBlobFormatVersion();
     createWorkers();
   }

From 7adcccf66f556db46adf00e6442953ef0f87b421 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Tue, 10 Oct 2023 14:40:34 +0200
Subject: [PATCH 293/356] SNOW-926149 Fix issues while using
 snowflake-jdbc-fips (#596)

---
 .github/workflows/End2EndTest.yml             |  23 +++
 README.md                                     |  19 ++
 e2e-jar-test/core/pom.xml                     |  38 ++++
 .../java/net/snowflake/IngestTestUtils.java   | 169 ++++++++++++++++++
 e2e-jar-test/fips/pom.xml                     |  53 ++++++
 .../java/net/snowflake/FipsIngestE2ETest.java |  31 ++++
 e2e-jar-test/pom.xml                          |  59 ++++++
 e2e-jar-test/run_e2e_jar_test.sh              |  61 +++++++
 e2e-jar-test/standard/pom.xml                 |  34 ++++
 .../net/snowflake/StandardIngestE2ETest.java  |  25 +++
 linkage-checker-exclusion-rules.xml           |  12 ++
 pom.xml                                       |  27 ++-
 scripts/process_licenses.py                   |   4 +
 scripts/update_project_version.py             |   2 +-
 .../net/snowflake/ingest/utils/ErrorCode.java |   3 +-
 .../net/snowflake/ingest/utils/Utils.java     |  66 +++++--
 .../ingest/ingest_error_messages.properties   |   3 +-
 .../java/net/snowflake/ingest/TestUtils.java  |   3 -
 .../streaming/internal/FlushServiceTest.java  |   7 -
 .../SnowflakeStreamingIngestClientTest.java   |  17 +-
 .../internal/datatypes/BinaryIT.java          |   2 +-
 21 files changed, 623 insertions(+), 35 deletions(-)
 create mode 100644 e2e-jar-test/core/pom.xml
 create mode 100644 e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java
 create mode 100644 e2e-jar-test/fips/pom.xml
 create mode 100644 e2e-jar-test/fips/src/test/java/net/snowflake/FipsIngestE2ETest.java
 create mode 100644 e2e-jar-test/pom.xml
 create mode 100755 e2e-jar-test/run_e2e_jar_test.sh
 create mode 100644 e2e-jar-test/standard/pom.xml
 create mode 100644 e2e-jar-test/standard/src/test/java/net/snowflake/StandardIngestE2ETest.java

diff --git a/.github/workflows/End2EndTest.yml b/.github/workflows/End2EndTest.yml
index a98c97fd2..91b16125e 100644
--- a/.github/workflows/End2EndTest.yml
+++ b/.github/workflows/End2EndTest.yml
@@ -67,3 +67,26 @@ jobs:
       - name: Unit & Integration Test (Windows)
         continue-on-error: false
         run: mvn -DghActionsIT verify --batch-mode
+  build-e2e-jar-test:
+    name: E2E JAR Test - ${{ matrix.java }}, Cloud ${{ matrix.snowflake_cloud }}
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        java: [ 8 ]
+        snowflake_cloud: [ 'AWS' ]
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+      - name: Install Java ${{ matrix.java }}
+        uses: actions/setup-java@v2
+        with:
+          distribution: temurin
+          java-version: ${{ matrix.java }}
+          cache: maven
+      - name: Decrypt profile.json for Cloud ${{ matrix.snowflake_cloud }}
+        env:
+          DECRYPTION_PASSPHRASE: ${{ secrets.PROFILE_JSON_DECRYPT_PASSPHRASE }}
+        run: ./scripts/decrypt_secret.sh ${{ matrix.snowflake_cloud }}
+      - name: Run E2E JAR Test
+        run: ./e2e-jar-test/run_e2e_jar_test.sh
diff --git a/README.md b/README.md
index edbded16d..0d745ba20 100644
--- a/README.md
+++ b/README.md
@@ -65,6 +65,25 @@ dependencies {
 }
 ```
 
+## Jar Versions
+
+The Snowflake Ingest SDK provides shaded and unshaded versions of its jar. The shaded version bundles the dependencies into its own jar,
+whereas the unshaded version declares its dependencies in `pom.xml`, which are fetched as standard transitive dependencies by the build system like Maven or Gradle.
+The shaded JAR can help avoid potential dependency conflicts, but the unshaded version provides finer graned control over transitive dependencies.
+
+## Using with snowflake-jdbc-fips
+
+For use cases, which need to use `snowflake-jdbc-fips` instead of the default `snowflake-jdbc`, we recommend to take the following steps:
+
+- Use the unshaded version of the Ingest SDK.
+- Exclude these transitive dependencies:
+    - `net.snowflake:snowflake-jdbc`
+    - `org.bouncycastle:bcpkix-jdk18on`
+    - `org.bouncycastle:bcprov-jdk18on`
+- Add a dependency on `snowflake-jdbc-fips`.
+
+See [this test](https://github.com/snowflakedb/snowflake-ingest-java/tree/master/e2e-jar-test/fips) for an example how to use Snowflake Ingest SDK together with Snowflake FIPS JDBC Driver.
+
 # Example
 
 ## Snowpipe
diff --git a/e2e-jar-test/core/pom.xml b/e2e-jar-test/core/pom.xml
new file mode 100644
index 000000000..36d52ae5b
--- /dev/null
+++ b/e2e-jar-test/core/pom.xml
@@ -0,0 +1,38 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>net.snowflake.snowflake-ingest-java-e2e-jar-test</groupId>
+        <artifactId>parent</artifactId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>core</artifactId>
+    <name>core</name>
+    <packaging>jar</packaging>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <!-- Provided because we let submodules define which way they pull in the SDK (e.g. with/without snowflake-jdbc-fips, etc) -->
+        <dependency>
+            <groupId>net.snowflake</groupId>
+            <artifactId>snowflake-ingest-sdk</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-simple</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+    </dependencies>
+</project>
diff --git a/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java b/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java
new file mode 100644
index 000000000..100972ea0
--- /dev/null
+++ b/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java
@@ -0,0 +1,169 @@
+package net.snowflake;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.KeyFactory;
+import java.security.NoSuchAlgorithmException;
+import java.security.PrivateKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.UUID;
+
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+
+public class IngestTestUtils {
+  private static final String PROFILE_PATH = "profile.json";
+
+  private final Connection connection;
+
+  private final String database;
+  private final String schema;
+  private final String table;
+
+  private final String testId;
+
+  private final SnowflakeStreamingIngestClient client;
+
+  private final SnowflakeStreamingIngestChannel channel;
+
+  private final ObjectMapper objectMapper = new ObjectMapper();
+
+  private final Random random = new Random();
+
+  private final Base64.Decoder base64Decoder = Base64.getDecoder();
+
+  public IngestTestUtils(String testName)
+      throws SQLException,
+          IOException,
+          ClassNotFoundException,
+          NoSuchAlgorithmException,
+          InvalidKeySpecException {
+    testId = String.format("%s_%s", testName, UUID.randomUUID().toString().replace("-", "_"));
+    connection = getConnection();
+    database = String.format("database_%s", testId);
+    schema = String.format("schema_%s", testId);
+    table = String.format("table_%s", testId);
+
+    connection.createStatement().execute(String.format("create database %s", database));
+    connection.createStatement().execute(String.format("create schema %s", schema));
+    connection.createStatement().execute(String.format("create table %s (c1 int, c2 varchar, c3 binary)", table));
+
+    client =
+        SnowflakeStreamingIngestClientFactory.builder("TestClient01")
+            .setProperties(loadProperties())
+            .build();
+
+    channel = client.openChannel(
+            OpenChannelRequest.builder(String.format("channel_%s", this.testId))
+                    .setDBName(database)
+                    .setSchemaName(schema)
+                    .setTableName(table)
+                    .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+                    .build());
+  }
+
+  private Properties loadProperties() throws IOException {
+    Properties props = new Properties();
+    Iterator<Map.Entry<String, JsonNode>> propIt =
+        objectMapper.readTree(new String(Files.readAllBytes(Paths.get(PROFILE_PATH)))).fields();
+    while (propIt.hasNext()) {
+      Map.Entry<String, JsonNode> prop = propIt.next();
+      props.put(prop.getKey(), prop.getValue().asText());
+    }
+    return props;
+  }
+
+  private Connection getConnection()
+      throws IOException, ClassNotFoundException, SQLException, NoSuchAlgorithmException, InvalidKeySpecException {
+    Class.forName("net.snowflake.client.jdbc.SnowflakeDriver");
+
+    Properties loadedProps = loadProperties();
+
+    byte[] decoded = base64Decoder.decode(loadedProps.getProperty("private_key"));
+    KeyFactory kf = KeyFactory.getInstance("RSA");
+
+    PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(decoded);
+    PrivateKey privateKey = kf.generatePrivate(keySpec);
+
+    Properties props = new Properties();
+    props.putAll(loadedProps);
+    props.put("client_session_keep_alive", "true");
+    props.put("privateKey", privateKey);
+
+    return DriverManager.getConnection(loadedProps.getProperty("connect_string"), props);
+  }
+
+  private Map<String, Object> createRow() {
+    Map<String, Object> row = new HashMap<>();
+
+    byte[] bytes = new byte[1024];
+    random.nextBytes(bytes);
+
+    row.put("c1", random.nextInt());
+    row.put("c2", String.valueOf(random.nextInt()));
+    row.put("c3", bytes);
+
+    return row;
+  }
+
+  /**
+   * Given a channel and expected offset, this method waits up to 60 seconds until the last
+   * committed offset is equal to the passed offset
+   */
+  private void waitForOffset(SnowflakeStreamingIngestChannel channel, String expectedOffset)
+          throws InterruptedException {
+    int counter = 0;
+    String lastCommittedOffset = null;
+    while (counter < 600) {
+      String currentOffset = channel.getLatestCommittedOffsetToken();
+      if (expectedOffset.equals(currentOffset)) {
+        return;
+      }
+      System.out.printf("Waiting for offset expected=%s actual=%s%n", expectedOffset, currentOffset);
+      lastCommittedOffset = currentOffset;
+      counter++;
+      Thread.sleep(100);
+    }
+    throw new RuntimeException(
+            String.format(
+                    "Timeout exceeded while waiting for offset %s. Last committed offset: %s",
+                    expectedOffset, lastCommittedOffset));
+  }
+
+  public void test() throws InterruptedException {
+    // Insert few rows one by one
+    for (int offset = 2; offset < 1000; offset++) {
+      offset++;
+      channel.insertRow(createRow(), String.valueOf(offset));
+    }
+
+    // Insert a batch of rows
+    String offset = "final-offset";
+    channel.insertRows(
+            Arrays.asList(createRow(), createRow(), createRow(), createRow(), createRow()), offset);
+
+    waitForOffset(channel, offset);
+  }
+
+  public void close() throws Exception {
+    connection.close();
+    channel.close().get();
+    client.close();
+  }
+}
diff --git a/e2e-jar-test/fips/pom.xml b/e2e-jar-test/fips/pom.xml
new file mode 100644
index 000000000..039af9b7a
--- /dev/null
+++ b/e2e-jar-test/fips/pom.xml
@@ -0,0 +1,53 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>net.snowflake.snowflake-ingest-java-e2e-jar-test</groupId>
+        <artifactId>parent</artifactId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>fips</artifactId>
+    <packaging>jar</packaging>
+    <name>fips</name>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>net.snowflake.snowflake-ingest-java-e2e-jar-test</groupId>
+            <artifactId>core</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>net.snowflake</groupId>
+            <artifactId>snowflake-ingest-sdk</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>net.snowflake</groupId>
+                    <artifactId>snowflake-jdbc</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.bouncycastle</groupId>
+                    <artifactId>bcpkix-jdk18on</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.bouncycastle</groupId>
+                    <artifactId>bcprov-jdk18on</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>net.snowflake</groupId>
+            <artifactId>snowflake-jdbc-fips</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+        </dependency>
+    </dependencies>
+</project>
diff --git a/e2e-jar-test/fips/src/test/java/net/snowflake/FipsIngestE2ETest.java b/e2e-jar-test/fips/src/test/java/net/snowflake/FipsIngestE2ETest.java
new file mode 100644
index 000000000..7279f23ff
--- /dev/null
+++ b/e2e-jar-test/fips/src/test/java/net/snowflake/FipsIngestE2ETest.java
@@ -0,0 +1,31 @@
+package net.snowflake;
+
+import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.security.Security;
+
+public class FipsIngestE2ETest {
+
+  private IngestTestUtils ingestTestUtils;
+
+  @Before
+  public void setUp() throws Exception {
+    // Add FIPS provider, the SDK does not do this by default
+    Security.addProvider(new BouncyCastleFipsProvider("C:HYBRID;ENABLE{All};"));
+
+    ingestTestUtils = new IngestTestUtils("fips_ingest");
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    ingestTestUtils.close();
+  }
+
+  @Test
+  public void name() throws InterruptedException {
+    ingestTestUtils.test();
+  }
+}
diff --git a/e2e-jar-test/pom.xml b/e2e-jar-test/pom.xml
new file mode 100644
index 000000000..9368308fc
--- /dev/null
+++ b/e2e-jar-test/pom.xml
@@ -0,0 +1,59 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>net.snowflake.snowflake-ingest-java-e2e-jar-test</groupId>
+  <artifactId>parent</artifactId>
+  <version>1.0-SNAPSHOT</version>
+  <packaging>pom</packaging>
+
+  <name>snowflake-ingest-sdk-e2e-test</name>
+
+  <modules>
+    <module>standard</module>
+    <module>fips</module>
+    <module>core</module>
+  </modules>
+
+  <dependencyManagement>
+
+    <dependencies>
+      <dependency>
+        <groupId>net.snowflake.snowflake-ingest-java-e2e-jar-test</groupId>
+        <artifactId>core</artifactId>
+        <version>${project.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>net.snowflake</groupId>
+        <artifactId>snowflake-ingest-sdk</artifactId>
+        <version>2.0.4-SNAPSHOT</version>
+      </dependency>
+
+      <dependency>
+        <groupId>net.snowflake</groupId>
+        <artifactId>snowflake-jdbc-fips</artifactId>
+        <version>3.13.30</version>
+      </dependency>
+
+      <dependency>
+        <groupId>org.slf4j</groupId>
+        <artifactId>slf4j-simple</artifactId>
+        <version>1.7.36</version>
+      </dependency>
+
+      <dependency>
+        <groupId>com.fasterxml.jackson.core</groupId>
+        <artifactId>jackson-databind</artifactId>
+        <version>2.14.0</version>
+      </dependency>
+
+      <dependency>
+        <groupId>junit</groupId>
+        <artifactId>junit</artifactId>
+        <version>4.13.2</version>
+        <scope>test</scope>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
+</project>
diff --git a/e2e-jar-test/run_e2e_jar_test.sh b/e2e-jar-test/run_e2e_jar_test.sh
new file mode 100755
index 000000000..e5e9b5c55
--- /dev/null
+++ b/e2e-jar-test/run_e2e_jar_test.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+## This script tests the SDK JARs end-to-end, i.e. not using integration tests from within the project, but from an
+## external Maven project, which depends on the SDK deployed into the local maven repository. The following SDK variants are tested:
+## 1. Shaded jar
+## 2. Unshaded jar
+## 3. FIPS-compliant jar, i.e. unshaded jar without snowflake-jdbc and bouncy castle dependencies, but with snowflake-jdbc-fips depedency
+
+maven_repo_dir=$(mvn help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)
+sdk_repo_dir="${maven_repo_dir}/net/snowflake/snowflake-ingest-sdk"
+
+cp profile.json e2e-jar-test/standard
+cp profile.json e2e-jar-test/fips
+
+###################
+# TEST SHADED JAR #
+###################
+
+# Remove the SDK from local maven repository
+rm -fr $sdk_repo_dir
+
+# Prepare pom.xml for shaded JAR
+project_version=$(./scripts/get_project_info_from_pom.py pom.xml version)
+./scripts/update_project_version.py public_pom.xml $project_version > generated_public_pom.xml
+
+# Build shaded SDK
+mvn clean package -DskipTests=true --batch-mode --show-version
+
+# Install shaded SDK JARs into local maven repository
+mvn install:install-file -Dfile=target/snowflake-ingest-sdk.jar -DpomFile=generated_public_pom.xml
+
+# Run e2e tests
+(cd e2e-jar-test && mvn clean verify -pl standard -am)
+
+#####################
+# TEST UNSHADED JAR #
+#####################
+
+# Remove the SDK from local maven repository
+rm -r $sdk_repo_dir
+
+# Install unshaded SDK into local maven repository
+mvn clean install -Dnot-shadeDep -DskipTests=true --batch-mode --show-version
+
+# Run e2e tests
+(cd e2e-jar-test && mvn clean verify -pl standard -am)
+
+#############
+# TEST FIPS #
+#############
+
+# Remove the SDK from local maven repository
+rm -r $sdk_repo_dir
+
+# Install unshaded SDK into local maven repository
+mvn clean install -Dnot-shadeDep -DskipTests=true --batch-mode --show-version
+
+# Run e2e tests on the FIPS module
+(cd e2e-jar-test && mvn clean verify -pl fips -am)
\ No newline at end of file
diff --git a/e2e-jar-test/standard/pom.xml b/e2e-jar-test/standard/pom.xml
new file mode 100644
index 000000000..2fdddcddd
--- /dev/null
+++ b/e2e-jar-test/standard/pom.xml
@@ -0,0 +1,34 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>net.snowflake.snowflake-ingest-java-e2e-jar-test</groupId>
+        <artifactId>parent</artifactId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>standard</artifactId>
+    <packaging>jar</packaging>
+    <name>unshaded</name>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>net.snowflake.snowflake-ingest-java-e2e-jar-test</groupId>
+            <artifactId>core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>net.snowflake</groupId>
+            <artifactId>snowflake-ingest-sdk</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+        </dependency>
+    </dependencies>
+</project>
diff --git a/e2e-jar-test/standard/src/test/java/net/snowflake/StandardIngestE2ETest.java b/e2e-jar-test/standard/src/test/java/net/snowflake/StandardIngestE2ETest.java
new file mode 100644
index 000000000..255577655
--- /dev/null
+++ b/e2e-jar-test/standard/src/test/java/net/snowflake/StandardIngestE2ETest.java
@@ -0,0 +1,25 @@
+package net.snowflake;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class StandardIngestE2ETest {
+
+  private IngestTestUtils ingestTestUtils;
+
+  @Before
+  public void setUp() throws Exception {
+    ingestTestUtils = new IngestTestUtils("standard_ingest");
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    ingestTestUtils.close();
+  }
+
+  @Test
+  public void name() throws InterruptedException {
+    ingestTestUtils.test();
+  }
+}
diff --git a/linkage-checker-exclusion-rules.xml b/linkage-checker-exclusion-rules.xml
index 3544c6d48..0cb2eb38c 100644
--- a/linkage-checker-exclusion-rules.xml
+++ b/linkage-checker-exclusion-rules.xml
@@ -19,6 +19,18 @@
         <Reason>Google Crypto Tink is an optional dependency of nimbus-jose-jwt</Reason>
     </LinkageError>
 
+    <LinkageError>
+        <Target>
+            <Class name="org.bouncycastle.asn1.x509.qualified.QCStatement" />
+        </Target>
+        <Reason>Seems like a false positive, this class does exist on classpath.</Reason>
+    </LinkageError>
+    <LinkageError>
+        <Target>
+            <Class name="org.bouncycastle.asn1.x9.X9FieldID" />
+        </Target>
+        <Reason>Seems like a false positive, this class does exist on classpath.</Reason>
+    </LinkageError>
     <!-- Hadoop-related linkage errors-->
     <LinkageError>
         <Source>
diff --git a/pom.xml b/pom.xml
index dbf156690..fc4a2ab81 100644
--- a/pom.xml
+++ b/pom.xml
@@ -35,6 +35,7 @@
 
   <!-- Set our Language Level to Java 8 -->
   <properties>
+    <bouncycastle.version>1.74</bouncycastle.version>
     <codehaus.version>1.9.13</codehaus.version>
     <commonscodec.version>1.15</commonscodec.version>
     <commonscollections.version>3.2.2</commonscollections.version>
@@ -274,6 +275,16 @@
         <artifactId>audience-annotations</artifactId>
         <version>${yetus.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.bouncycastle</groupId>
+        <artifactId>bcpkix-jdk18on</artifactId>
+        <version>${bouncycastle.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.bouncycastle</groupId>
+        <artifactId>bcprov-jdk18on</artifactId>
+        <version>${bouncycastle.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.codehaus.jackson</groupId>
         <artifactId>jackson-core-asl</artifactId>
@@ -449,6 +460,15 @@
       <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-hadoop</artifactId>
     </dependency>
+
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcpkix-jdk18on</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcprov-jdk18on</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
@@ -531,7 +551,7 @@
             <dependency>
               <groupId>com.google.cloud.tools</groupId>
               <artifactId>linkage-checker-enforcer-rules</artifactId>
-              <version>1.5.12</version>
+              <version>1.5.13</version>
             </dependency>
             <dependency>
               <groupId>org.codehaus.mojo</groupId>
@@ -758,6 +778,7 @@
             <includedLicense>The MIT License</includedLicense>
             <includedLicense>EDL 1.0</includedLicense>
             <includedLicense>The Go license</includedLicense>
+            <includedLicense>Bouncy Castle Licence</includedLicense>
           </includedLicenses>
           <excludedScopes>test,provided,system</excludedScopes>
           <failOnBlacklist>true</failOnBlacklist>
@@ -929,6 +950,10 @@
                   <pattern>com.nimbusds</pattern>
                   <shadedPattern>${shadeBase}.com.nimbusds</shadedPattern>
                 </relocation>
+                <relocation>
+                  <pattern>org.bouncycastle</pattern>
+                  <shadedPattern>${shadeBase}.org.bouncycastle</shadedPattern>
+                </relocation>
                 <relocation>
                   <pattern>net.jcip</pattern>
                   <shadedPattern>${shadeBase}.net.jcip</shadedPattern>
diff --git a/scripts/process_licenses.py b/scripts/process_licenses.py
index a17c34843..bb43fbbf0 100644
--- a/scripts/process_licenses.py
+++ b/scripts/process_licenses.py
@@ -30,6 +30,7 @@
 EDL_10_LICENSE = "EDL 1.0"
 MIT_LICENSE = "The MIT License"
 GO_LICENSE = "The Go license"
+BOUNCY_CASTLE_LICENSE = "Bouncy Castle Licence <https://www.bouncycastle.org/licence.html>"
 
 # The SDK does not need to include licenses of dependencies, which aren't shaded
 IGNORED_DEPENDENCIES = {"net.snowflake:snowflake-jdbc", "org.slf4j:slf4j-api"}
@@ -57,6 +58,9 @@
     "org.apache.parquet:parquet-format-structures": APACHE_LICENSE,
     "com.github.luben:zstd-jni": BSD_2_CLAUSE_LICENSE,
     "io.airlift:aircompressor": APACHE_LICENSE,
+    "org.bouncycastle:bcpkix-jdk18on": BOUNCY_CASTLE_LICENSE,
+    "org.bouncycastle:bcutil-jdk18on": BOUNCY_CASTLE_LICENSE,
+    "org.bouncycastle:bcprov-jdk18on": BOUNCY_CASTLE_LICENSE,
 }
 
 
diff --git a/scripts/update_project_version.py b/scripts/update_project_version.py
index dd844a7c5..c67d58802 100755
--- a/scripts/update_project_version.py
+++ b/scripts/update_project_version.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 # -*- coding: utf-8 -*-
 #
 # Update project version
diff --git a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
index 97bdae0f1..b212d7244 100644
--- a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
+++ b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
@@ -40,7 +40,8 @@ public enum ErrorCode {
   MAKE_URI_FAILURE("0032"),
   OAUTH_REFRESH_TOKEN_ERROR("0033"),
   INVALID_CONFIG_PARAMETER("0034"),
-  MAX_BATCH_SIZE_EXCEEDED("0035");
+  MAX_BATCH_SIZE_EXCEEDED("0035"),
+  CRYPTO_PROVIDER_ERROR("0036");
 
   public static final String errorMessageResource = "net.snowflake.ingest.ingest_error_messages";
 
diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java
index ca39fed0d..a2653ab14 100644
--- a/src/main/java/net/snowflake/ingest/utils/Utils.java
+++ b/src/main/java/net/snowflake/ingest/utils/Utils.java
@@ -12,10 +12,12 @@
 import java.lang.management.BufferPoolMXBean;
 import java.lang.management.ManagementFactory;
 import java.lang.management.MemoryUsage;
+import java.lang.reflect.InvocationTargetException;
 import java.security.KeyFactory;
 import java.security.KeyPair;
 import java.security.NoSuchAlgorithmException;
 import java.security.PrivateKey;
+import java.security.Provider;
 import java.security.PublicKey;
 import java.security.Security;
 import java.security.interfaces.RSAPrivateCrtKey;
@@ -26,20 +28,66 @@
 import java.util.Map;
 import java.util.Properties;
 import net.snowflake.client.core.SFSessionProperty;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.PEMParser;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.operator.InputDecryptorProvider;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo;
 import org.apache.commons.codec.binary.Base64;
+import org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
+import org.bouncycastle.openssl.PEMParser;
+import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;
+import org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder;
+import org.bouncycastle.operator.InputDecryptorProvider;
+import org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo;
 
 /** Contains Ingest related utility functions */
 public class Utils {
 
   private static final Logging logger = new Logging(Utils.class);
 
+  private static final String DEFAULT_SECURITY_PROVIDER_NAME =
+      "org.bouncycastle.jce.provider.BouncyCastleProvider";
+
+  /** provider name */
+  private static final String BOUNCY_CASTLE_PROVIDER = "BC";
+  /** provider name for FIPS */
+  private static final String BOUNCY_CASTLE_FIPS_PROVIDER = "BCFIPS";
+
+  static {
+    // Add Bouncy Castle to the security provider. This is required to
+    // verify the signature on OCSP response and attached certificates.
+    if (Security.getProvider(BOUNCY_CASTLE_PROVIDER) == null
+        && Security.getProvider(BOUNCY_CASTLE_FIPS_PROVIDER) == null) {
+      Security.addProvider(instantiateSecurityProvider());
+    }
+  }
+
+  public static Provider getProvider() {
+    final Provider bcProvider = Security.getProvider(BOUNCY_CASTLE_PROVIDER);
+    if (bcProvider != null) {
+      return bcProvider;
+    }
+    final Provider bcFipsProvider = Security.getProvider(BOUNCY_CASTLE_FIPS_PROVIDER);
+    if (bcFipsProvider != null) {
+      return bcFipsProvider;
+    }
+    throw new SFException(ErrorCode.INTERNAL_ERROR, "No security provider found");
+  }
+
+  private static Provider instantiateSecurityProvider() {
+    try {
+      logger.logInfo("Adding security provider {}", DEFAULT_SECURITY_PROVIDER_NAME);
+      Class klass = Class.forName(DEFAULT_SECURITY_PROVIDER_NAME);
+      return (Provider) klass.getDeclaredConstructor().newInstance();
+    } catch (ExceptionInInitializerError
+        | ClassNotFoundException
+        | NoSuchMethodException
+        | InstantiationException
+        | IllegalAccessException
+        | IllegalArgumentException
+        | InvocationTargetException
+        | SecurityException ex) {
+      throw new SFException(
+          ErrorCode.CRYPTO_PROVIDER_ERROR, DEFAULT_SECURITY_PROVIDER_NAME, ex.getMessage());
+    }
+  }
+
   /**
    * Assert when the String is null or Empty
    *
@@ -183,7 +231,6 @@ public static PrivateKey parsePrivateKey(String key) {
     key = key.replaceAll("-+[A-Za-z ]+-+", "");
     key = key.replaceAll("\\s", "");
 
-    java.security.Security.addProvider(new BouncyCastleProvider());
     byte[] encoded = Base64.decodeBase64(key);
     try {
       KeyFactory kf = KeyFactory.getInstance("RSA");
@@ -216,7 +263,6 @@ public static PrivateKey parseEncryptedPrivateKey(String key, String passphrase)
     }
     builder.append("\n-----END ENCRYPTED PRIVATE KEY-----");
     key = builder.toString();
-    Security.addProvider(new BouncyCastleProvider());
     try {
       PEMParser pemParser = new PEMParser(new StringReader(key));
       PKCS8EncryptedPrivateKeyInfo encryptedPrivateKeyInfo =
@@ -225,7 +271,7 @@ public static PrivateKey parseEncryptedPrivateKey(String key, String passphrase)
       InputDecryptorProvider pkcs8Prov =
           new JceOpenSSLPKCS8DecryptorProviderBuilder().build(passphrase.toCharArray());
       JcaPEMKeyConverter converter =
-          new JcaPEMKeyConverter().setProvider(BouncyCastleProvider.PROVIDER_NAME);
+          new JcaPEMKeyConverter().setProvider(Utils.getProvider().getName());
       PrivateKeyInfo decryptedPrivateKeyInfo =
           encryptedPrivateKeyInfo.decryptPrivateKeyInfo(pkcs8Prov);
       return converter.getPrivateKey(decryptedPrivateKeyInfo);
diff --git a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
index 9698799f7..268afd051 100644
--- a/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
+++ b/src/main/resources/net/snowflake/ingest/ingest_error_messages.properties
@@ -37,4 +37,5 @@
 0032=URI builder fail to build url: {0}
 0033=OAuth token refresh failure: {0}
 0034=Invalid config parameter: {0}
-0035=Too large batch of rows passed to insertRows, the batch size cannot exceed {0} bytes, recommended batch size for optimal performance and memory utilization is {1} bytes. We recommend splitting large batches into multiple smaller ones and call insertRows for each smaller batch separately.
\ No newline at end of file
+0035=Too large batch of rows passed to insertRows, the batch size cannot exceed {0} bytes, recommended batch size for optimal performance and memory utilization is {1} bytes. We recommend splitting large batches into multiple smaller ones and call insertRows for each smaller batch separately.
+0036=Failed to load {0}. If you use FIPS, import BouncyCastleFipsProvider in the application: {1}
\ No newline at end of file
diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java
index b2ebe1bdd..ba14ff610 100644
--- a/src/test/java/net/snowflake/ingest/TestUtils.java
+++ b/src/test/java/net/snowflake/ingest/TestUtils.java
@@ -41,7 +41,6 @@
 import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.utils.Constants;
@@ -123,8 +122,6 @@ private static void init() throws Exception {
       role = Optional.ofNullable(profile.get(ROLE)).map(r -> r.asText()).orElse("DEFAULT_ROLE");
       privateKeyPem = profile.get(PRIVATE_KEY).asText();
 
-      java.security.Security.addProvider(new BouncyCastleProvider());
-
       byte[] encoded = Base64.decodeBase64(privateKeyPem);
       KeyFactory kf = KeyFactory.getInstance("RSA");
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 0b8e8b4cd..5a77a51db 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -39,7 +39,6 @@
 import javax.crypto.BadPaddingException;
 import javax.crypto.IllegalBlockSizeException;
 import javax.crypto.NoSuchPaddingException;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.Cryptor;
@@ -47,7 +46,6 @@
 import net.snowflake.ingest.utils.ParameterProvider;
 import net.snowflake.ingest.utils.SFException;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 import org.mockito.ArgumentMatchers;
@@ -277,11 +275,6 @@ TestContext<List<List<Object>>> create() {
 
   TestContextFactory<?> testContextFactory;
 
-  @Before
-  public void setup() {
-    java.security.Security.addProvider(new BouncyCastleProvider());
-  }
-
   private SnowflakeStreamingIngestChannelInternal<?> addChannel1(TestContext<?> testContext) {
     return testContext
         .channelBuilder("channel1")
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index ec9f671bb..a99054c9f 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -19,7 +19,6 @@
 import java.io.StringWriter;
 import java.security.KeyPair;
 import java.security.PrivateKey;
-import java.security.Security;
 import java.time.ZoneOffset;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -40,13 +39,6 @@
 import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost;
 import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.asn1.nist.NISTObjectIdentifiers;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.jce.provider.BouncyCastleProvider;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.openssl.jcajce.JcaPEMWriter;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.operator.OperatorCreationException;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfoBuilder;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.jcajce.JcaPKCS8EncryptedPrivateKeyInfoBuilder;
-import net.snowflake.client.jdbc.internal.org.bouncycastle.pkcs.jcajce.JcePKCSPBEOutputEncryptorBuilder;
 import net.snowflake.ingest.TestUtils;
 import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
@@ -59,6 +51,12 @@
 import net.snowflake.ingest.utils.SFException;
 import net.snowflake.ingest.utils.SnowflakeURL;
 import net.snowflake.ingest.utils.Utils;
+import org.bouncycastle.asn1.nist.NISTObjectIdentifiers;
+import org.bouncycastle.openssl.jcajce.JcaPEMWriter;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfoBuilder;
+import org.bouncycastle.pkcs.jcajce.JcaPKCS8EncryptedPrivateKeyInfoBuilder;
+import org.bouncycastle.pkcs.jcajce.JcePKCSPBEOutputEncryptorBuilder;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -293,7 +291,6 @@ public void testEncryptedPrivateKey() throws Exception {
 
   private String generateAESKey(PrivateKey key, char[] passwd)
       throws IOException, OperatorCreationException {
-    Security.addProvider(new BouncyCastleProvider());
     StringWriter writer = new StringWriter();
     JcaPEMWriter pemWriter = new JcaPEMWriter(writer);
     PKCS8EncryptedPrivateKeyInfoBuilder pkcs8EncryptedPrivateKeyInfoBuilder =
@@ -301,7 +298,7 @@ private String generateAESKey(PrivateKey key, char[] passwd)
     pemWriter.writeObject(
         pkcs8EncryptedPrivateKeyInfoBuilder.build(
             new JcePKCSPBEOutputEncryptorBuilder(NISTObjectIdentifiers.id_aes256_CBC)
-                .setProvider(BouncyCastleProvider.PROVIDER_NAME)
+                .setProvider(Utils.getProvider().getName())
                 .build(passwd)));
     pemWriter.close();
     return writer.toString();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
index d9e6e3ddd..2a5023a3a 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/BinaryIT.java
@@ -1,6 +1,6 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
-import net.snowflake.client.jdbc.internal.org.bouncycastle.util.encoders.Hex;
+import org.bouncycastle.util.encoders.Hex;
 import org.junit.Test;
 
 public class BinaryIT extends AbstractDataTypeTest {

From e4a87ddc0866c3479b8957409b3bdeac906cb31a Mon Sep 17 00:00:00 2001
From: Xin Huang <xin.huang@snowflake.com>
Date: Wed, 11 Oct 2023 14:55:25 -0700
Subject: [PATCH 294/356] Update readme for JDBC version issues #599 (#600)

Update readme for https://github.com/snowflakedb/snowflake-ingest-java/issues/599
---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 0d745ba20..5027f3216 100644
--- a/README.md
+++ b/README.md
@@ -15,7 +15,7 @@ authentication. Currently, we support ingestion through the following APIs:
 
 The Snowflake Ingest Service SDK depends on the following libraries:
 
-* snowflake-jdbc (3.13.30+)
+* snowflake-jdbc (3.13.30 to 3.13.33) 
 * slf4j-api
 
 These dependencies will be fetched automatically by build systems like Maven or Gradle. If you don't build your project
@@ -23,7 +23,7 @@ using a build system, please make sure these dependencies are on the classpath.
 
 # Prerequisites
 
-**If your project depends on the Snowflake JDBC driver, as well, please make sure the JDBC driver version is 3.13.30 or newer.**
+**If your project depends on the Snowflake JDBC driver, as well, please make sure the JDBC driver version is 3.13.30 to 3.13.33. JDBC driver version 3.14.0 or higher is currently not supported. **
 
 ## Java 8+
 

From 92540de1e22e9b9539538e2b90636abaf298578e Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Fri, 13 Oct 2023 16:57:29 -0700
Subject: [PATCH 295/356] SNOW-906224: Add new ON_ERROR=SKIP_BATCH option
 (#597)

Introduce a new ON_ERROR option SKIP_BATCH

- Skips the entire batch
- Returns all the erroneous rows as part of the response, along with their indexes and error messages

Pros:
- Easy implementation

Cons:
- Worse performance: data validation will be done twice for the good rows
---
 .../ingest/streaming/OpenChannelRequest.java  |   2 +
 .../streaming/internal/AbstractRowBuffer.java | 202 +++++++++++++-----
 .../streaming/internal/IngestionStrategy.java |  21 ++
 .../streaming/internal/ParquetRowBuffer.java  |   3 +-
 .../streaming/internal/RowBufferTest.java     |  99 ++++++++-
 .../SnowflakeStreamingIngestChannelTest.java  |  24 +++
 .../streaming/internal/StreamingIngestIT.java |  57 +++++
 7 files changed, 342 insertions(+), 66 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/IngestionStrategy.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java b/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java
index c72629e94..f48288330 100644
--- a/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java
+++ b/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java
@@ -12,6 +12,8 @@ public class OpenChannelRequest {
   public enum OnErrorOption {
     CONTINUE, // CONTINUE loading the rows, and return all the errors in the response
     ABORT, // ABORT the entire batch, and throw an exception when we hit the first error
+    SKIP_BATCH, // If an error in the batch is detected return a response containing all error row
+    // indexes. No data is ingested
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index dde639d00..30be56fcb 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -138,6 +138,132 @@ public int getOrdinal() {
     }
   }
 
+  /** Insert rows function strategy for ON_ERROR=CONTINUE */
+  public class ContinueIngestionStrategy<T> implements IngestionStrategy<T> {
+    @Override
+    public InsertValidationResponse insertRows(
+        AbstractRowBuffer<T> rowBuffer, Iterable<Map<String, Object>> rows, String offsetToken) {
+      InsertValidationResponse response = new InsertValidationResponse();
+      float rowsSizeInBytes = 0F;
+      int rowIndex = 0;
+      for (Map<String, Object> row : rows) {
+        InsertValidationResponse.InsertError error =
+            new InsertValidationResponse.InsertError(row, rowIndex);
+        try {
+          Set<String> inputColumnNames = verifyInputColumns(row, error, rowIndex);
+          rowsSizeInBytes +=
+              addRow(
+                  row, rowBuffer.bufferedRowCount, rowBuffer.statsMap, inputColumnNames, rowIndex);
+          rowBuffer.bufferedRowCount++;
+        } catch (SFException e) {
+          error.setException(e);
+          response.addError(error);
+        } catch (Throwable e) {
+          logger.logWarn("Unexpected error happens during insertRows: {}", e);
+          error.setException(new SFException(e, ErrorCode.INTERNAL_ERROR, e.getMessage()));
+          response.addError(error);
+        }
+        checkBatchSizeEnforcedMaximum(rowsSizeInBytes);
+        rowIndex++;
+        if (rowBuffer.bufferedRowCount == Integer.MAX_VALUE) {
+          throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
+        }
+      }
+      checkBatchSizeRecommendedMaximum(rowsSizeInBytes);
+      rowBuffer.channelState.setOffsetToken(offsetToken);
+      rowBuffer.bufferSize += rowsSizeInBytes;
+      rowBuffer.rowSizeMetric.accept(rowsSizeInBytes);
+      return response;
+    }
+  }
+
+  /** Insert rows function strategy for ON_ERROR=ABORT */
+  public class AbortIngestionStrategy<T> implements IngestionStrategy<T> {
+    @Override
+    public InsertValidationResponse insertRows(
+        AbstractRowBuffer<T> rowBuffer, Iterable<Map<String, Object>> rows, String offsetToken) {
+      // If the on_error option is ABORT, simply throw the first exception
+      InsertValidationResponse response = new InsertValidationResponse();
+      float rowsSizeInBytes = 0F;
+      float tempRowsSizeInBytes = 0F;
+      int tempRowCount = 0;
+      for (Map<String, Object> row : rows) {
+        Set<String> inputColumnNames = verifyInputColumns(row, null, tempRowCount);
+        tempRowsSizeInBytes +=
+            addTempRow(row, tempRowCount, rowBuffer.tempStatsMap, inputColumnNames, tempRowCount);
+        checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes);
+        tempRowCount++;
+      }
+      checkBatchSizeRecommendedMaximum(tempRowsSizeInBytes);
+
+      moveTempRowsToActualBuffer(tempRowCount);
+
+      rowsSizeInBytes = tempRowsSizeInBytes;
+      if ((long) rowBuffer.bufferedRowCount + tempRowCount >= Integer.MAX_VALUE) {
+        throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
+      }
+      rowBuffer.bufferedRowCount += tempRowCount;
+      rowBuffer.statsMap.forEach(
+          (colName, stats) ->
+              rowBuffer.statsMap.put(
+                  colName,
+                  RowBufferStats.getCombinedStats(stats, rowBuffer.tempStatsMap.get(colName))));
+      rowBuffer.channelState.setOffsetToken(offsetToken);
+      rowBuffer.bufferSize += rowsSizeInBytes;
+      rowBuffer.rowSizeMetric.accept(rowsSizeInBytes);
+      return response;
+    }
+  }
+
+  /** Insert rows function strategy for ON_ERROR=SKIP_BATCH */
+  public class SkipBatchIngestionStrategy<T> implements IngestionStrategy<T> {
+    @Override
+    public InsertValidationResponse insertRows(
+        AbstractRowBuffer<T> rowBuffer, Iterable<Map<String, Object>> rows, String offsetToken) {
+      InsertValidationResponse response = new InsertValidationResponse();
+      float rowsSizeInBytes = 0F;
+      float tempRowsSizeInBytes = 0F;
+      int tempRowCount = 0;
+      for (Map<String, Object> row : rows) {
+        InsertValidationResponse.InsertError error =
+            new InsertValidationResponse.InsertError(row, tempRowCount);
+        try {
+          Set<String> inputColumnNames = verifyInputColumns(row, error, tempRowCount);
+          tempRowsSizeInBytes +=
+              addTempRow(row, tempRowCount, rowBuffer.tempStatsMap, inputColumnNames, tempRowCount);
+          tempRowCount++;
+        } catch (SFException e) {
+          error.setException(e);
+          response.addError(error);
+        } catch (Throwable e) {
+          logger.logWarn("Unexpected error happens during insertRows: {}", e);
+          error.setException(new SFException(e, ErrorCode.INTERNAL_ERROR, e.getMessage()));
+          response.addError(error);
+        }
+        checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes);
+      }
+
+      if (!response.hasErrors()) {
+        checkBatchSizeRecommendedMaximum(tempRowsSizeInBytes);
+        moveTempRowsToActualBuffer(tempRowCount);
+        rowsSizeInBytes = tempRowsSizeInBytes;
+        if ((long) rowBuffer.bufferedRowCount + tempRowCount >= Integer.MAX_VALUE) {
+          throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
+        }
+        rowBuffer.bufferedRowCount += tempRowCount;
+        rowBuffer.statsMap.forEach(
+            (colName, stats) ->
+                rowBuffer.statsMap.put(
+                    colName,
+                    RowBufferStats.getCombinedStats(stats, rowBuffer.tempStatsMap.get(colName))));
+        rowBuffer.channelState.setOffsetToken(offsetToken);
+      }
+      rowBuffer.bufferSize += rowsSizeInBytes;
+      rowBuffer.rowSizeMetric.accept(rowsSizeInBytes);
+      return response;
+    }
+  }
+
   // Map the column name to the stats
   @VisibleForTesting Map<String, RowBufferStats> statsMap;
 
@@ -297,76 +423,20 @@ Set<String> verifyInputColumns(
   @Override
   public InsertValidationResponse insertRows(
       Iterable<Map<String, Object>> rows, String offsetToken) {
-    float rowsSizeInBytes = 0F;
     if (!hasColumns()) {
       throw new SFException(ErrorCode.INTERNAL_ERROR, "Empty column fields");
     }
-    InsertValidationResponse response = new InsertValidationResponse();
+    InsertValidationResponse response = null;
     this.flushLock.lock();
     try {
       this.channelState.updateInsertStats(System.currentTimeMillis(), this.bufferedRowCount);
-      if (onErrorOption == OpenChannelRequest.OnErrorOption.CONTINUE) {
-        // Used to map incoming row(nth row) to InsertError(for nth row) in response
-        int rowIndex = 0;
-        for (Map<String, Object> row : rows) {
-          InsertValidationResponse.InsertError error =
-              new InsertValidationResponse.InsertError(row, rowIndex);
-          try {
-            Set<String> inputColumnNames = verifyInputColumns(row, error, rowIndex);
-            rowsSizeInBytes +=
-                addRow(row, this.bufferedRowCount, this.statsMap, inputColumnNames, rowIndex);
-            this.bufferedRowCount++;
-          } catch (SFException e) {
-            error.setException(e);
-            response.addError(error);
-          } catch (Throwable e) {
-            logger.logWarn("Unexpected error happens during insertRows: {}", e);
-            error.setException(new SFException(e, ErrorCode.INTERNAL_ERROR, e.getMessage()));
-            response.addError(error);
-          }
-          checkBatchSizeEnforcedMaximum(rowsSizeInBytes);
-          rowIndex++;
-          if (this.bufferedRowCount == Integer.MAX_VALUE) {
-            throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
-          }
-        }
-        checkBatchSizeRecommendedMaximum(rowsSizeInBytes);
-      } else {
-        // If the on_error option is ABORT, simply throw the first exception
-        float tempRowsSizeInBytes = 0F;
-        int tempRowCount = 0;
-        for (Map<String, Object> row : rows) {
-          Set<String> inputColumnNames = verifyInputColumns(row, null, tempRowCount);
-          tempRowsSizeInBytes +=
-              addTempRow(row, tempRowCount, this.tempStatsMap, inputColumnNames, tempRowCount);
-          checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes);
-          tempRowCount++;
-        }
-        checkBatchSizeRecommendedMaximum(tempRowsSizeInBytes);
-
-        moveTempRowsToActualBuffer(tempRowCount);
-
-        rowsSizeInBytes = tempRowsSizeInBytes;
-        if ((long) this.bufferedRowCount + tempRowCount >= Integer.MAX_VALUE) {
-          throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
-        }
-        this.bufferedRowCount += tempRowCount;
-        this.statsMap.forEach(
-            (colName, stats) ->
-                this.statsMap.put(
-                    colName,
-                    RowBufferStats.getCombinedStats(stats, this.tempStatsMap.get(colName))));
-      }
-
-      this.bufferSize += rowsSizeInBytes;
-      this.channelState.setOffsetToken(offsetToken);
-      this.rowSizeMetric.accept(rowsSizeInBytes);
+      IngestionStrategy<T> ingestionStrategy = createIngestionStrategy(onErrorOption);
+      response = ingestionStrategy.insertRows(this, rows, offsetToken);
     } finally {
       this.tempStatsMap.values().forEach(RowBufferStats::reset);
       clearTempRows();
       this.flushLock.unlock();
     }
-
     return response;
   }
 
@@ -581,4 +651,18 @@ private void checkBatchSizeRecommendedMaximum(float batchSizeInBytes) {
           INSERT_ROWS_RECOMMENDED_MAX_BATCH_SIZE_IN_BYTES);
     }
   }
+
+  /** Create the ingestion strategy based on the channel on_error option */
+  IngestionStrategy<T> createIngestionStrategy(OpenChannelRequest.OnErrorOption onErrorOption) {
+    switch (onErrorOption) {
+      case CONTINUE:
+        return new ContinueIngestionStrategy<>();
+      case ABORT:
+        return new AbortIngestionStrategy<>();
+      case SKIP_BATCH:
+        return new SkipBatchIngestionStrategy<>();
+      default:
+        throw new IllegalArgumentException("Unknown on error option: " + onErrorOption);
+    }
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/IngestionStrategy.java b/src/main/java/net/snowflake/ingest/streaming/internal/IngestionStrategy.java
new file mode 100644
index 000000000..defac31c4
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/IngestionStrategy.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2023 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import java.util.Map;
+import net.snowflake.ingest.streaming.InsertValidationResponse;
+
+/** Interface to a batch of rows into the row buffer based on different on error options */
+public interface IngestionStrategy<T> {
+  /**
+   * Insert a batch of rows into the row buffer
+   *
+   * @param rows input row
+   * @param offsetToken offset token of the latest row in the batch
+   * @return insert response that possibly contains errors because of insertion failures
+   */
+  InsertValidationResponse insertRows(
+      AbstractRowBuffer<T> rowBuffer, Iterable<Map<String, Object>> rows, String offsetToken);
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 75966eb35..6fb7b8027 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -97,7 +97,8 @@ public void setupSchema(List<ColumnMetadata> columns) {
       this.statsMap.put(
           column.getInternalName(), new RowBufferStats(column.getName(), column.getCollation()));
 
-      if (onErrorOption == OpenChannelRequest.OnErrorOption.ABORT) {
+      if (onErrorOption == OpenChannelRequest.OnErrorOption.ABORT
+          || onErrorOption == OpenChannelRequest.OnErrorOption.SKIP_BATCH) {
         this.tempStatsMap.put(
             column.getInternalName(), new RowBufferStats(column.getName(), column.getCollation()));
       }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 8d71d9a44..549eefbde 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -30,6 +30,7 @@ public class RowBufferTest {
   private final boolean enableParquetMemoryOptimization;
   private AbstractRowBuffer<?> rowBufferOnErrorContinue;
   private AbstractRowBuffer<?> rowBufferOnErrorAbort;
+  private AbstractRowBuffer<?> rowBufferOnErrorSkipBatch;
 
   public RowBufferTest() {
     this.enableParquetMemoryOptimization = false;
@@ -39,9 +40,11 @@ public RowBufferTest() {
   public void setupRowBuffer() {
     this.rowBufferOnErrorContinue = createTestBuffer(OpenChannelRequest.OnErrorOption.CONTINUE);
     this.rowBufferOnErrorAbort = createTestBuffer(OpenChannelRequest.OnErrorOption.ABORT);
+    this.rowBufferOnErrorSkipBatch = createTestBuffer(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
     List<ColumnMetadata> schema = createSchema();
     this.rowBufferOnErrorContinue.setupSchema(schema);
     this.rowBufferOnErrorAbort.setupSchema(schema);
+    this.rowBufferOnErrorSkipBatch.setupSchema(schema);
   }
 
   static List<ColumnMetadata> createSchema() {
@@ -467,6 +470,7 @@ private void testFlushHelper(AbstractRowBuffer<?> rowBuffer) {
   public void testDoubleQuotesColumnName() {
     testDoubleQuotesColumnNameHelper(OpenChannelRequest.OnErrorOption.ABORT);
     testDoubleQuotesColumnNameHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+    testDoubleQuotesColumnNameHelper(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
   }
 
   private void testDoubleQuotesColumnNameHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
@@ -739,6 +743,7 @@ private void testStatsE2EHelper(AbstractRowBuffer<?> rowBuffer) {
   public void testStatsE2ETimestamp() {
     testStatsE2ETimestampHelper(OpenChannelRequest.OnErrorOption.ABORT);
     testStatsE2ETimestampHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+    testStatsE2ETimestampHelper(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
   }
 
   private void testStatsE2ETimestampHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
@@ -820,6 +825,7 @@ private void testStatsE2ETimestampHelper(OpenChannelRequest.OnErrorOption onErro
   public void testE2EDate() {
     testE2EDateHelper(OpenChannelRequest.OnErrorOption.ABORT);
     testE2EDateHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+    testE2EDateHelper(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
   }
 
   private void testE2EDateHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
@@ -868,6 +874,7 @@ private void testE2EDateHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
   public void testE2ETime() {
     testE2ETimeHelper(OpenChannelRequest.OnErrorOption.ABORT);
     testE2ETimeHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+    testE2ETimeHelper(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
   }
 
   private void testE2ETimeHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
@@ -940,6 +947,7 @@ private void testE2ETimeHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
   public void testMaxInsertRowsBatchSize() {
     testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
     testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption.ABORT);
+    testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
   }
 
   private void testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
@@ -977,6 +985,7 @@ private void testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption o
   public void testNullableCheck() {
     testNullableCheckHelper(OpenChannelRequest.OnErrorOption.ABORT);
     testNullableCheckHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+    testNullableCheckHelper(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
   }
 
   private void testNullableCheckHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
@@ -1016,6 +1025,7 @@ private void testNullableCheckHelper(OpenChannelRequest.OnErrorOption onErrorOpt
   public void testMissingColumnCheck() {
     testMissingColumnCheckHelper(OpenChannelRequest.OnErrorOption.ABORT);
     testMissingColumnCheckHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+    testMissingColumnCheckHelper(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
   }
 
   private void testMissingColumnCheckHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
@@ -1087,13 +1097,10 @@ public void testExtraColumnsCheck() {
   }
 
   @Test
-  public void testFailureHalfwayThroughColumnProcessingAbort() {
-    doTestFailureHalfwayThroughColumnProcessing(OpenChannelRequest.OnErrorOption.ABORT);
-  }
-
-  @Test
-  public void testFailureHalfwayThroughColumnProcessingContinue() {
+  public void testFailureHalfwayThroughColumnProcessing() {
     doTestFailureHalfwayThroughColumnProcessing(OpenChannelRequest.OnErrorOption.CONTINUE);
+    doTestFailureHalfwayThroughColumnProcessing(OpenChannelRequest.OnErrorOption.ABORT);
+    doTestFailureHalfwayThroughColumnProcessing(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
   }
 
   private void doTestFailureHalfwayThroughColumnProcessing(
@@ -1170,6 +1177,7 @@ private void doTestFailureHalfwayThroughColumnProcessing(
   public void testE2EBoolean() {
     testE2EBooleanHelper(OpenChannelRequest.OnErrorOption.ABORT);
     testE2EBooleanHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+    testE2EBooleanHelper(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
   }
 
   private void testE2EBooleanHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
@@ -1218,6 +1226,7 @@ private void testE2EBooleanHelper(OpenChannelRequest.OnErrorOption onErrorOption
   public void testE2EBinary() {
     testE2EBinaryHelper(OpenChannelRequest.OnErrorOption.ABORT);
     testE2EBinaryHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+    testE2EBinaryHelper(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
   }
 
   private void testE2EBinaryHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
@@ -1274,6 +1283,7 @@ private void testE2EBinaryHelper(OpenChannelRequest.OnErrorOption onErrorOption)
   public void testE2EReal() {
     testE2ERealHelper(OpenChannelRequest.OnErrorOption.ABORT);
     testE2ERealHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+    testE2ERealHelper(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
   }
 
   private void testE2ERealHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
@@ -1393,10 +1403,83 @@ public void testOnErrorAbortFailures() {
     Assert.assertEquals(0, innerBuffer.bufferedRowCount);
   }
 
+  @Test
+  public void testOnErrorAbortSkipBatch() {
+    AbstractRowBuffer<?> innerBuffer =
+        createTestBuffer(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
+
+    ColumnMetadata colDecimal = new ColumnMetadata();
+    colDecimal.setName("COLDECIMAL");
+    colDecimal.setPhysicalType("SB16");
+    colDecimal.setNullable(true);
+    colDecimal.setLogicalType("FIXED");
+    colDecimal.setPrecision(38);
+    colDecimal.setScale(0);
+
+    innerBuffer.setupSchema(Collections.singletonList(colDecimal));
+    Map<String, Object> row = new HashMap<>();
+    row.put("COLDECIMAL", 1);
+
+    InsertValidationResponse response = innerBuffer.insertRows(Collections.singletonList(row), "1");
+    Assert.assertFalse(response.hasErrors());
+
+    Assert.assertEquals(1, innerBuffer.bufferedRowCount);
+    Assert.assertEquals(0, innerBuffer.getTempRowCount());
+    Assert.assertEquals(
+        1, innerBuffer.statsMap.get("COLDECIMAL").getCurrentMaxIntValue().intValue());
+    Assert.assertEquals(
+        1, innerBuffer.statsMap.get("COLDECIMAL").getCurrentMinIntValue().intValue());
+    Assert.assertNull(innerBuffer.tempStatsMap.get("COLDECIMAL").getCurrentMaxIntValue());
+    Assert.assertNull(innerBuffer.tempStatsMap.get("COLDECIMAL").getCurrentMinIntValue());
+
+    Map<String, Object> row2 = new HashMap<>();
+    row2.put("COLDECIMAL", 2);
+    Map<String, Object> row3 = new HashMap<>();
+    row3.put("COLDECIMAL", true);
+
+    response = innerBuffer.insertRows(Arrays.asList(row2, row3), "3");
+    Assert.assertTrue(response.hasErrors());
+
+    Assert.assertEquals(1, innerBuffer.bufferedRowCount);
+    Assert.assertEquals(0, innerBuffer.getTempRowCount());
+    Assert.assertEquals(
+        1, innerBuffer.statsMap.get("COLDECIMAL").getCurrentMaxIntValue().intValue());
+    Assert.assertEquals(
+        1, innerBuffer.statsMap.get("COLDECIMAL").getCurrentMinIntValue().intValue());
+    Assert.assertNull(innerBuffer.tempStatsMap.get("COLDECIMAL").getCurrentMaxIntValue());
+    Assert.assertNull(innerBuffer.tempStatsMap.get("COLDECIMAL").getCurrentMinIntValue());
+
+    Assert.assertEquals(1, innerBuffer.bufferedRowCount);
+    Assert.assertEquals(0, innerBuffer.getTempRowCount());
+    Assert.assertEquals(
+        1, innerBuffer.statsMap.get("COLDECIMAL").getCurrentMaxIntValue().intValue());
+    Assert.assertEquals(
+        1, innerBuffer.statsMap.get("COLDECIMAL").getCurrentMinIntValue().intValue());
+    Assert.assertNull(innerBuffer.tempStatsMap.get("COLDECIMAL").getCurrentMaxIntValue());
+    Assert.assertNull(innerBuffer.tempStatsMap.get("COLDECIMAL").getCurrentMinIntValue());
+
+    row3.put("COLDECIMAL", 3);
+    response = innerBuffer.insertRows(Arrays.asList(row2, row3), "3");
+    Assert.assertFalse(response.hasErrors());
+    Assert.assertEquals(3, innerBuffer.bufferedRowCount);
+    Assert.assertEquals(0, innerBuffer.getTempRowCount());
+    Assert.assertEquals(
+        3, innerBuffer.statsMap.get("COLDECIMAL").getCurrentMaxIntValue().intValue());
+    Assert.assertEquals(
+        1, innerBuffer.statsMap.get("COLDECIMAL").getCurrentMinIntValue().intValue());
+    Assert.assertNull(innerBuffer.tempStatsMap.get("COLDECIMAL").getCurrentMaxIntValue());
+    Assert.assertNull(innerBuffer.tempStatsMap.get("COLDECIMAL").getCurrentMinIntValue());
+
+    ChannelData<?> data = innerBuffer.flush("my_snowpipe_streaming.bdec");
+    Assert.assertEquals(3, data.getRowCount());
+    Assert.assertEquals(0, innerBuffer.bufferedRowCount);
+  }
+
   @Test
   public void testE2EVariant() {
     testE2EVariantHelper(OpenChannelRequest.OnErrorOption.ABORT);
     testE2EVariantHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+    testE2EVariantHelper(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
   }
 
   private void testE2EVariantHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
@@ -1446,6 +1529,7 @@ private void testE2EVariantHelper(OpenChannelRequest.OnErrorOption onErrorOption
   public void testE2EObject() {
     testE2EObjectHelper(OpenChannelRequest.OnErrorOption.ABORT);
     testE2EObjectHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+    testE2EObjectHelper(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
   }
 
   private void testE2EObjectHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
@@ -1477,6 +1561,7 @@ private void testE2EObjectHelper(OpenChannelRequest.OnErrorOption onErrorOption)
   public void testE2EArray() {
     testE2EArrayHelper(OpenChannelRequest.OnErrorOption.ABORT);
     testE2EArrayHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
+    testE2EArrayHelper(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
   }
 
   private void testE2EArrayHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
@@ -1527,6 +1612,8 @@ public void testOnErrorAbortRowsWithError() {
         createTestBuffer(OpenChannelRequest.OnErrorOption.CONTINUE);
     AbstractRowBuffer<?> innerBufferOnErrorAbort =
         createTestBuffer(OpenChannelRequest.OnErrorOption.ABORT);
+    AbstractRowBuffer<?> innerBufferOnErrorSkipBatch =
+        createTestBuffer(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
 
     ColumnMetadata colChar = new ColumnMetadata();
     colChar.setName("COLCHAR");
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 8fbf67264..a1e26b63d 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -645,6 +645,30 @@ public void testInsertTooLargeRow() {
       Assert.assertEquals(ErrorCode.MAX_ROW_SIZE_EXCEEDED.getMessageCode(), e.getVendorCode());
       Assert.assertEquals(expectedMessage, e.getMessage());
     }
+
+    // Test channel with on error SKIP_BATCH
+    channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
+            "channel",
+            "db",
+            "schema",
+            "table",
+            "0",
+            0L,
+            0L,
+            client,
+            "key",
+            1234L,
+            OpenChannelRequest.OnErrorOption.SKIP_BATCH,
+            UTC);
+    channel.setupSchema(schema);
+
+    insertValidationResponse = channel.insertRow(row, "token-1");
+    Assert.assertEquals(1, insertValidationResponse.getErrorRowCount());
+    thrownException = insertValidationResponse.getInsertErrors().get(0).getException();
+    Assert.assertEquals(
+        ErrorCode.MAX_ROW_SIZE_EXCEEDED.getMessageCode(), thrownException.getVendorCode());
+    Assert.assertEquals(expectedMessage, thrownException.getMessage());
   }
 
   @Test
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index dd7ae7aa8..95cc5c699 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -860,6 +860,63 @@ public void testAbortOnErrorOption() throws Exception {
     Assert.fail("Row sequencer not updated before timeout");
   }
 
+  @Test
+  public void testAbortOnErrorSkipBatch() throws Exception {
+    String onErrorOptionTable = "skip_batch_on_error_option";
+    jdbcConnection
+        .createStatement()
+        .execute(String.format("create or replace table %s (c1 int);", onErrorOptionTable));
+
+    OpenChannelRequest request =
+        OpenChannelRequest.builder("CHANNEL")
+            .setDBName(testDb)
+            .setSchemaName(TEST_SCHEMA)
+            .setTableName(onErrorOptionTable)
+            .setOnErrorOption(OpenChannelRequest.OnErrorOption.SKIP_BATCH)
+            .build();
+
+    // Open a streaming ingest channel from the given client
+    SnowflakeStreamingIngestChannel channel = client.openChannel(request);
+    Map<String, Object> row1 = new HashMap<>();
+    row1.put("c1", 1);
+    Map<String, Object> row2 = new HashMap<>();
+    row2.put("c1", 2);
+    Map<String, Object> row3 = new HashMap<>();
+    row3.put("c1", "a");
+
+    verifyInsertValidationResponse(channel.insertRow(row1, "1"));
+
+    InsertValidationResponse response = channel.insertRows(Arrays.asList(row1, row2, row3), "3");
+    Assert.assertTrue(response.hasErrors());
+
+    Map<String, Object> row4 = new HashMap<>();
+    row4.put("c1", 4);
+    verifyInsertValidationResponse(channel.insertRow(row4, "4"));
+
+    // Close the channel after insertion
+    channel.close().get();
+
+    for (int i = 1; i < 15; i++) {
+      if (channel.getLatestCommittedOffsetToken() != null
+          && channel.getLatestCommittedOffsetToken().equals("4")) {
+        ResultSet result =
+            jdbcConnection
+                .createStatement()
+                .executeQuery(
+                    String.format(
+                        "select count(c1), min(c1), max(c1) from %s.%s.%s",
+                        testDb, TEST_SCHEMA, onErrorOptionTable));
+        result.next();
+        Assert.assertEquals("2", result.getString(1));
+        Assert.assertEquals("1", result.getString(2));
+        Assert.assertEquals("4", result.getString(3));
+        return;
+      }
+      Thread.sleep(1000);
+    }
+    Assert.fail("Row sequencer not updated before timeout");
+  }
+
   @Test
   public void testChannelClose() throws Exception {
     OpenChannelRequest request1 =

From 9d5602af9238f8a950b8014704c0c30de16674b7 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Tue, 17 Oct 2023 12:47:55 -0700
Subject: [PATCH 296/356] NO-SNOW: Fix two issues in insertRows API (#602)

This PR contains two fixes in the current insertRows logic:

- We can't thrown any exception when ON_ERROR=CONTINUE, since it will cause duplicate data if we insert rows into the buffer but not updating the offset token
- Update the row buffer only if the rows are inserted successfully for ON_ERROR=SKIP_BATCH
---
 .../streaming/internal/AbstractRowBuffer.java | 25 +++++++++----------
 .../streaming/internal/RowBufferTest.java     |  1 -
 2 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 30be56fcb..3029e7a49 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -150,6 +150,9 @@ public InsertValidationResponse insertRows(
         InsertValidationResponse.InsertError error =
             new InsertValidationResponse.InsertError(row, rowIndex);
         try {
+          if (rowBuffer.bufferedRowCount == Integer.MAX_VALUE) {
+            throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
+          }
           Set<String> inputColumnNames = verifyInputColumns(row, error, rowIndex);
           rowsSizeInBytes +=
               addRow(
@@ -163,11 +166,7 @@ public InsertValidationResponse insertRows(
           error.setException(new SFException(e, ErrorCode.INTERNAL_ERROR, e.getMessage()));
           response.addError(error);
         }
-        checkBatchSizeEnforcedMaximum(rowsSizeInBytes);
         rowIndex++;
-        if (rowBuffer.bufferedRowCount == Integer.MAX_VALUE) {
-          throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
-        }
       }
       checkBatchSizeRecommendedMaximum(rowsSizeInBytes);
       rowBuffer.channelState.setOffsetToken(offsetToken);
@@ -191,17 +190,17 @@ public InsertValidationResponse insertRows(
         Set<String> inputColumnNames = verifyInputColumns(row, null, tempRowCount);
         tempRowsSizeInBytes +=
             addTempRow(row, tempRowCount, rowBuffer.tempStatsMap, inputColumnNames, tempRowCount);
-        checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes);
         tempRowCount++;
+        checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes);
+        if ((long) rowBuffer.bufferedRowCount + tempRowCount >= Integer.MAX_VALUE) {
+          throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
+        }
       }
       checkBatchSizeRecommendedMaximum(tempRowsSizeInBytes);
 
       moveTempRowsToActualBuffer(tempRowCount);
 
       rowsSizeInBytes = tempRowsSizeInBytes;
-      if ((long) rowBuffer.bufferedRowCount + tempRowCount >= Integer.MAX_VALUE) {
-        throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
-      }
       rowBuffer.bufferedRowCount += tempRowCount;
       rowBuffer.statsMap.forEach(
           (colName, stats) ->
@@ -241,15 +240,15 @@ public InsertValidationResponse insertRows(
           response.addError(error);
         }
         checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes);
+        if ((long) rowBuffer.bufferedRowCount + tempRowCount >= Integer.MAX_VALUE) {
+          throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
+        }
       }
 
       if (!response.hasErrors()) {
         checkBatchSizeRecommendedMaximum(tempRowsSizeInBytes);
         moveTempRowsToActualBuffer(tempRowCount);
         rowsSizeInBytes = tempRowsSizeInBytes;
-        if ((long) rowBuffer.bufferedRowCount + tempRowCount >= Integer.MAX_VALUE) {
-          throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
-        }
         rowBuffer.bufferedRowCount += tempRowCount;
         rowBuffer.statsMap.forEach(
             (colName, stats) ->
@@ -257,9 +256,9 @@ public InsertValidationResponse insertRows(
                     colName,
                     RowBufferStats.getCombinedStats(stats, rowBuffer.tempStatsMap.get(colName))));
         rowBuffer.channelState.setOffsetToken(offsetToken);
+        rowBuffer.bufferSize += rowsSizeInBytes;
+        rowBuffer.rowSizeMetric.accept(rowsSizeInBytes);
       }
-      rowBuffer.bufferSize += rowsSizeInBytes;
-      rowBuffer.rowSizeMetric.accept(rowsSizeInBytes);
       return response;
     }
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 549eefbde..269b6f28d 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -945,7 +945,6 @@ private void testE2ETimeHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
 
   @Test
   public void testMaxInsertRowsBatchSize() {
-    testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption.CONTINUE);
     testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption.ABORT);
     testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
   }

From ed7ba380c54ed53f95494eba6b115c91d54a9864 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Wed, 18 Oct 2023 13:22:58 +0200
Subject: [PATCH 297/356] SNOW-902709 Limit the max allowed number of chunks in
 blob (#580)

---
 .../streaming/internal/FlushService.java      |  12 ++
 ...nowflakeStreamingIngestClientInternal.java |  48 ++++-
 .../ingest/utils/ParameterProvider.java       |  21 ++
 .../streaming/internal/FlushServiceTest.java  | 119 +++++++++++-
 .../streaming/internal/ManyTablesIT.java      |  98 ++++++++++
 .../internal/ParameterProviderTest.java       |   9 +
 .../SnowflakeStreamingIngestClientTest.java   | 180 +++++++++++++-----
 7 files changed, 439 insertions(+), 48 deletions(-)
 create mode 100644 src/test/java/net/snowflake/ingest/streaming/internal/ManyTablesIT.java

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index ff879b8d6..cb1ef7810 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -350,6 +350,18 @@ void distributeFlushTasks() {
         if (!leftoverChannelsDataPerTable.isEmpty()) {
           channelsDataPerTable.addAll(leftoverChannelsDataPerTable);
           leftoverChannelsDataPerTable.clear();
+        } else if (blobData.size()
+            >= this.owningClient
+                .getParameterProvider()
+                .getMaxChunksInBlobAndRegistrationRequest()) {
+          // Create a new blob if the current one already contains max allowed number of chunks
+          logger.logInfo(
+              "Max allowed number of chunks in the current blob reached. chunkCount={}"
+                  + " maxChunkCount={} currentBlobPath={}",
+              blobData.size(),
+              this.owningClient.getParameterProvider().getMaxChunksInBlobAndRegistrationRequest(),
+              blobPath);
+          break;
         } else {
           ConcurrentHashMap<String, SnowflakeStreamingIngestChannelInternal<T>> table =
               itr.next().getValue();
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 5916e472d..27ff9407e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -452,7 +452,53 @@ ChannelsStatusResponse getChannelsStatus(
    * @param blobs list of uploaded blobs
    */
   void registerBlobs(List<BlobMetadata> blobs) {
-    this.registerBlobs(blobs, 0);
+    for (List<BlobMetadata> blobBatch : partitionBlobListForRegistrationRequest(blobs)) {
+      this.registerBlobs(blobBatch, 0);
+    }
+  }
+
+  /**
+   * Partition the collection of blobs into sub-lists, so that the total number of chunks in each
+   * sublist does not exceed the max allowed number of chunks in one registration request.
+   */
+  List<List<BlobMetadata>> partitionBlobListForRegistrationRequest(List<BlobMetadata> blobs) {
+    List<List<BlobMetadata>> result = new ArrayList<>();
+    List<BlobMetadata> currentBatch = new ArrayList<>();
+    int chunksInCurrentBatch = 0;
+    int maxChunksInBlobAndRegistrationRequest =
+        parameterProvider.getMaxChunksInBlobAndRegistrationRequest();
+
+    for (BlobMetadata blob : blobs) {
+      if (blob.getChunks().size() > maxChunksInBlobAndRegistrationRequest) {
+        throw new SFException(
+            ErrorCode.INTERNAL_ERROR,
+            String.format(
+                "Incorrectly generated blob detected - number of chunks in the blob is larger than"
+                    + " the max allowed number of chunks. Please report this bug to Snowflake."
+                    + " bdec=%s chunkCount=%d maxAllowedChunkCount=%d",
+                blob.getPath(), blob.getChunks().size(), maxChunksInBlobAndRegistrationRequest));
+      }
+
+      if (chunksInCurrentBatch + blob.getChunks().size() > maxChunksInBlobAndRegistrationRequest) {
+        // Newly added BDEC file would exceed the max number of chunks in a single registration
+        // request. We put chunks collected so far into the result list and create a new batch with
+        // the current blob
+        result.add(currentBatch);
+        currentBatch = new ArrayList<>();
+        currentBatch.add(blob);
+        chunksInCurrentBatch = blob.getChunks().size();
+      } else {
+        // Newly added BDEC can be added to the current batch because it does not exceed the max
+        // number of chunks in a single registration request, yet.
+        currentBatch.add(blob);
+        chunksInCurrentBatch += blob.getChunks().size();
+      }
+    }
+
+    if (!currentBatch.isEmpty()) {
+      result.add(currentBatch);
+    }
+    return result;
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
index 5c6f81f66..3a2221697 100644
--- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
+++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -31,6 +31,8 @@ public class ParameterProvider {
   public static final String MAX_CHUNK_SIZE_IN_BYTES = "MAX_CHUNK_SIZE_IN_BYTES".toLowerCase();
   public static final String MAX_ALLOWED_ROW_SIZE_IN_BYTES =
       "MAX_ALLOWED_ROW_SIZE_IN_BYTES".toLowerCase();
+  public static final String MAX_CHUNKS_IN_BLOB_AND_REGISTRATION_REQUEST =
+      "MAX_CHUNKS_IN_BLOB_AND_REGISTRATION_REQUEST".toLowerCase();
 
   public static final String MAX_CLIENT_LAG = "MAX_CLIENT_LAG".toLowerCase();
 
@@ -59,6 +61,7 @@ public class ParameterProvider {
 
   static final long MAX_CLIENT_LAG_MS_MAX = TimeUnit.MINUTES.toMillis(10);
   public static final long MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT = 64 * 1024 * 1024; // 64 MB
+  public static final int MAX_CHUNKS_IN_BLOB_AND_REGISTRATION_REQUEST_DEFAULT = 100;
 
   /* Parameter that enables using internal Parquet buffers for buffering of rows before serializing.
   It reduces memory consumption compared to using Java Objects for buffering.*/
@@ -170,6 +173,11 @@ private void setParameterMap(Map<String, Object> parameterOverrides, Properties
     this.updateValue(MAX_CLIENT_LAG, MAX_CLIENT_LAG_DEFAULT, parameterOverrides, props);
     this.updateValue(
         MAX_CLIENT_LAG_ENABLED, MAX_CLIENT_LAG_ENABLED_DEFAULT, parameterOverrides, props);
+    this.updateValue(
+        MAX_CHUNKS_IN_BLOB_AND_REGISTRATION_REQUEST,
+        MAX_CHUNKS_IN_BLOB_AND_REGISTRATION_REQUEST_DEFAULT,
+        parameterOverrides,
+        props);
   }
 
   /** @return Longest interval in milliseconds between buffer flushes */
@@ -369,6 +377,7 @@ public long getMaxChunkSizeInBytes() {
     return (val instanceof String) ? Long.parseLong(val.toString()) : (long) val;
   }
 
+  /** @return The max allow row size (in bytes) */
   public long getMaxAllowedRowSizeInBytes() {
     Object val =
         this.parameterMap.getOrDefault(
@@ -376,6 +385,18 @@ public long getMaxAllowedRowSizeInBytes() {
     return (val instanceof String) ? Long.parseLong(val.toString()) : (long) val;
   }
 
+  /**
+   * @return The max number of chunks that can be put into a single BDEC or blob registration
+   *     request.
+   */
+  public int getMaxChunksInBlobAndRegistrationRequest() {
+    Object val =
+        this.parameterMap.getOrDefault(
+            MAX_CHUNKS_IN_BLOB_AND_REGISTRATION_REQUEST,
+            MAX_CHUNKS_IN_BLOB_AND_REGISTRATION_REQUEST_DEFAULT);
+    return (val instanceof String) ? Integer.parseInt(val.toString()) : (int) val;
+  }
+
   @Override
   public String toString() {
     return "ParameterProvider{" + "parameterMap=" + parameterMap + '}';
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 5a77a51db..17f929206 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -35,6 +35,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TimeZone;
+import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import javax.crypto.BadPaddingException;
 import javax.crypto.IllegalBlockSizeException;
@@ -273,7 +274,22 @@ TestContext<List<List<Object>>> create() {
     }
   }
 
-  TestContextFactory<?> testContextFactory;
+  TestContextFactory<List<List<Object>>> testContextFactory;
+
+  private SnowflakeStreamingIngestChannelInternal<List<List<Object>>> addChannel(
+      TestContext<List<List<Object>>> testContext, int tableId, long encryptionKeyId) {
+    return testContext
+        .channelBuilder("channel" + UUID.randomUUID())
+        .setDBName("db1")
+        .setSchemaName("PUBLIC")
+        .setTableName("table" + tableId)
+        .setOffsetToken("offset1")
+        .setChannelSequencer(0L)
+        .setRowSequencer(0L)
+        .setEncryptionKey("key")
+        .setEncryptionKeyId(encryptionKeyId)
+        .buildAndAdd();
+  }
 
   private SnowflakeStreamingIngestChannelInternal<?> addChannel1(TestContext<?> testContext) {
     return testContext
@@ -546,6 +562,107 @@ public void testBlobSplitDueToChunkSizeLimit() throws Exception {
     Mockito.verify(flushService, Mockito.times(2)).buildAndUpload(Mockito.any(), Mockito.any());
   }
 
+  @Test
+  public void testBlobSplitDueToNumberOfChunks() throws Exception {
+    for (int rowCount : Arrays.asList(0, 1, 30, 111, 159, 287, 1287, 1599, 4496)) {
+      runTestBlobSplitDueToNumberOfChunks(rowCount);
+    }
+  }
+
+  /**
+   * Insert rows in batches of 3 into each table and assert that the expected number of blobs is
+   * generated.
+   *
+   * @param numberOfRows How many rows to insert
+   */
+  public void runTestBlobSplitDueToNumberOfChunks(int numberOfRows) throws Exception {
+    int channelsPerTable = 3;
+    int expectedBlobs =
+        (int)
+            Math.ceil(
+                (double) numberOfRows
+                    / channelsPerTable
+                    / ParameterProvider.MAX_CHUNKS_IN_BLOB_AND_REGISTRATION_REQUEST_DEFAULT);
+
+    final TestContext<List<List<Object>>> testContext = testContextFactory.create();
+
+    for (int i = 0; i < numberOfRows; i++) {
+      SnowflakeStreamingIngestChannelInternal<List<List<Object>>> channel =
+          addChannel(testContext, i / channelsPerTable, 1);
+      channel.setupSchema(Collections.singletonList(createLargeTestTextColumn("C1")));
+      channel.insertRow(Collections.singletonMap("C1", i), "");
+    }
+
+    FlushService<List<List<Object>>> flushService = testContext.flushService;
+    flushService.flush(true).get();
+
+    ArgumentCaptor<List<List<ChannelData<List<List<Object>>>>>> blobDataCaptor =
+        ArgumentCaptor.forClass(List.class);
+    Mockito.verify(flushService, Mockito.times(expectedBlobs))
+        .buildAndUpload(Mockito.any(), blobDataCaptor.capture());
+
+    // 1. list => blobs; 2. list => chunks; 3. list => channels; 4. list => rows, 5. list => columns
+    List<List<List<ChannelData<List<List<Object>>>>>> allUploadedBlobs =
+        blobDataCaptor.getAllValues();
+
+    Assert.assertEquals(numberOfRows, getRows(allUploadedBlobs).size());
+  }
+
+  @Test
+  public void testBlobSplitDueToNumberOfChunksWithLeftoverChannels() throws Exception {
+    final TestContext<List<List<Object>>> testContext = testContextFactory.create();
+
+    for (int i = 0; i < 99; i++) { // 19 simple chunks
+      SnowflakeStreamingIngestChannelInternal<List<List<Object>>> channel =
+          addChannel(testContext, i, 1);
+      channel.setupSchema(Collections.singletonList(createLargeTestTextColumn("C1")));
+      channel.insertRow(Collections.singletonMap("C1", i), "");
+    }
+
+    // 20th chunk would contain multiple channels, but there are some with different encryption key
+    // ID, so they spill to a new blob
+    SnowflakeStreamingIngestChannelInternal<List<List<Object>>> channel1 =
+        addChannel(testContext, 99, 1);
+    channel1.setupSchema(Collections.singletonList(createLargeTestTextColumn("C1")));
+    channel1.insertRow(Collections.singletonMap("C1", 0), "");
+
+    SnowflakeStreamingIngestChannelInternal<List<List<Object>>> channel2 =
+        addChannel(testContext, 99, 2);
+    channel2.setupSchema(Collections.singletonList(createLargeTestTextColumn("C1")));
+    channel2.insertRow(Collections.singletonMap("C1", 0), "");
+
+    SnowflakeStreamingIngestChannelInternal<List<List<Object>>> channel3 =
+        addChannel(testContext, 99, 2);
+    channel3.setupSchema(Collections.singletonList(createLargeTestTextColumn("C1")));
+    channel3.insertRow(Collections.singletonMap("C1", 0), "");
+
+    FlushService<List<List<Object>>> flushService = testContext.flushService;
+    flushService.flush(true).get();
+
+    ArgumentCaptor<List<List<ChannelData<List<List<Object>>>>>> blobDataCaptor =
+        ArgumentCaptor.forClass(List.class);
+    Mockito.verify(flushService, Mockito.atLeast(2))
+        .buildAndUpload(Mockito.any(), blobDataCaptor.capture());
+
+    // 1. list => blobs; 2. list => chunks; 3. list => channels; 4. list => rows, 5. list => columns
+    List<List<List<ChannelData<List<List<Object>>>>>> allUploadedBlobs =
+        blobDataCaptor.getAllValues();
+
+    Assert.assertEquals(102, getRows(allUploadedBlobs).size());
+  }
+
+  private List<List<Object>> getRows(List<List<List<ChannelData<List<List<Object>>>>>> blobs) {
+    List<List<Object>> result = new ArrayList<>();
+    blobs.forEach(
+        chunks ->
+            chunks.forEach(
+                channels ->
+                    channels.forEach(
+                        chunkData ->
+                            result.addAll(((ParquetChunkData) chunkData.getVectors()).rows))));
+    return result;
+  }
+
   @Test
   public void testBuildAndUpload() throws Exception {
     long expectedBuildLatencyMs = 100;
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ManyTablesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/ManyTablesIT.java
new file mode 100644
index 000000000..d32adfe3f
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ManyTablesIT.java
@@ -0,0 +1,98 @@
+package net.snowflake.ingest.streaming.internal;
+
+import static net.snowflake.ingest.utils.Constants.ROLE;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Properties;
+import net.snowflake.ingest.TestUtils;
+import net.snowflake.ingest.streaming.OpenChannelRequest;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
+import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import net.snowflake.ingest.utils.Constants;
+import net.snowflake.ingest.utils.ParameterProvider;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Verified that ingestion work when we ingest into large number of tables from the same client and
+ * blobs and registration requests have to be cut, so they don't contain large number of chunks
+ */
+public class ManyTablesIT {
+
+  private static final int TABLES_COUNT = 20;
+  private static final int TOTAL_ROWS_COUNT = 200_000;
+  private String dbName;
+  private SnowflakeStreamingIngestClient client;
+  private Connection connection;
+  private SnowflakeStreamingIngestChannel[] channels;
+  private String[] offsetTokensPerChannel;
+
+  @Before
+  public void setUp() throws Exception {
+    Properties props = TestUtils.getProperties(Constants.BdecVersion.THREE, false);
+    props.put(ParameterProvider.MAX_CHUNKS_IN_BLOB_AND_REGISTRATION_REQUEST, 2);
+    if (props.getProperty(ROLE).equals("DEFAULT_ROLE")) {
+      props.setProperty(ROLE, "ACCOUNTADMIN");
+    }
+    client = SnowflakeStreamingIngestClientFactory.builder("client1").setProperties(props).build();
+    connection = TestUtils.getConnection(true);
+    dbName = String.format("sdk_it_many_tables_db_%d", System.nanoTime());
+
+    channels = new SnowflakeStreamingIngestChannel[TABLES_COUNT];
+    offsetTokensPerChannel = new String[TABLES_COUNT];
+    connection.createStatement().execute(String.format("create database %s;", dbName));
+
+    String[] tableNames = new String[TABLES_COUNT];
+    for (int i = 0; i < tableNames.length; i++) {
+      tableNames[i] = String.format("table_%d", i);
+      connection.createStatement().execute(String.format("create table table_%d(c int);", i));
+      channels[i] =
+          client.openChannel(
+              OpenChannelRequest.builder(String.format("channel-%d", i))
+                  .setDBName(dbName)
+                  .setSchemaName("public")
+                  .setTableName(tableNames[i])
+                  .setOnErrorOption(OpenChannelRequest.OnErrorOption.ABORT)
+                  .build());
+    }
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    connection.createStatement().execute(String.format("drop database %s;", dbName));
+    client.close();
+    connection.close();
+  }
+
+  @Test
+  public void testIngestionIntoManyTables() throws InterruptedException, SQLException {
+    for (int i = 0; i < TOTAL_ROWS_COUNT; i++) {
+      Map<String, Object> row = Collections.singletonMap("c", i);
+      String offset = String.valueOf(i);
+      int channelId = i % channels.length;
+      channels[channelId].insertRow(row, offset);
+      offsetTokensPerChannel[channelId] = offset;
+    }
+
+    for (int i = 0; i < channels.length; i++) {
+      TestUtils.waitForOffset(channels[i], offsetTokensPerChannel[i]);
+    }
+
+    int totalRowsCount = 0;
+    ResultSet rs =
+        connection
+            .createStatement()
+            .executeQuery(String.format("show tables in database %s;", dbName));
+    while (rs.next()) {
+      totalRowsCount += rs.getInt("rows");
+    }
+    Assert.assertEquals(TOTAL_ROWS_COUNT, totalRowsCount);
+  }
+}
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
index def5f7ecf..7838763de 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
@@ -272,4 +272,13 @@ public void testMaxClientLagEnabledThresholdAbove() {
       Assert.assertTrue(e.getMessage().startsWith("Lag falls outside"));
     }
   }
+
+  @Test
+  public void testMaxChunksInBlobAndRegistrationRequest() {
+    Properties prop = new Properties();
+    Map<String, Object> parameterMap = getStartingParameterMap();
+    parameterMap.put("max_chunks_in_blob_and_registration_request", 1);
+    ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+    Assert.assertEquals(1, parameterProvider.getMaxChunksInBlobAndRegistrationRequest());
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index a99054c9f..11fc0b93b 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -11,6 +11,8 @@
 import static net.snowflake.ingest.utils.Constants.ROLE;
 import static net.snowflake.ingest.utils.Constants.USER;
 import static net.snowflake.ingest.utils.ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.when;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.PropertyAccessor;
@@ -319,11 +321,11 @@ public void testGetChannelsStatusWithRequest() throws Exception {
     CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
-    Mockito.when(statusLine.getStatusCode()).thenReturn(200);
-    Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine);
-    Mockito.when(httpResponse.getEntity()).thenReturn(httpEntity);
-    Mockito.when(httpEntity.getContent()).thenReturn(IOUtils.toInputStream(responseString));
-    Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
+    when(statusLine.getStatusCode()).thenReturn(200);
+    when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    when(httpResponse.getEntity()).thenReturn(httpEntity);
+    when(httpEntity.getContent()).thenReturn(IOUtils.toInputStream(responseString));
+    when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
 
     RequestBuilder requestBuilder =
         Mockito.spy(
@@ -378,11 +380,11 @@ public void testGetChannelsStatusWithRequestError() throws Exception {
     CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
-    Mockito.when(statusLine.getStatusCode()).thenReturn(500);
-    Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine);
-    Mockito.when(httpResponse.getEntity()).thenReturn(httpEntity);
-    Mockito.when(httpEntity.getContent()).thenReturn(IOUtils.toInputStream(responseString));
-    Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
+    when(statusLine.getStatusCode()).thenReturn(500);
+    when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    when(httpResponse.getEntity()).thenReturn(httpEntity);
+    when(httpEntity.getContent()).thenReturn(IOUtils.toInputStream(responseString));
+    when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
 
     RequestBuilder requestBuilder =
         Mockito.spy(
@@ -645,12 +647,12 @@ public void testRegisterBlobErrorResponse() throws Exception {
     CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
-    Mockito.when(statusLine.getStatusCode()).thenReturn(500);
-    Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine);
-    Mockito.when(httpResponse.getEntity()).thenReturn(httpEntity);
+    when(statusLine.getStatusCode()).thenReturn(500);
+    when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    when(httpResponse.getEntity()).thenReturn(httpEntity);
     String response = "testRegisterBlobErrorResponse";
-    Mockito.when(httpEntity.getContent()).thenReturn(IOUtils.toInputStream(response));
-    Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
+    when(httpEntity.getContent()).thenReturn(IOUtils.toInputStream(response));
+    when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
 
     RequestBuilder requestBuilder =
         new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair());
@@ -693,11 +695,11 @@ public void testRegisterBlobSnowflakeInternalErrorResponse() throws Exception {
     CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
-    Mockito.when(statusLine.getStatusCode()).thenReturn(200);
-    Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine);
-    Mockito.when(httpResponse.getEntity()).thenReturn(httpEntity);
-    Mockito.when(httpEntity.getContent()).thenReturn(IOUtils.toInputStream(response));
-    Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
+    when(statusLine.getStatusCode()).thenReturn(200);
+    when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    when(httpResponse.getEntity()).thenReturn(httpEntity);
+    when(httpEntity.getContent()).thenReturn(IOUtils.toInputStream(response));
+    when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
 
     RequestBuilder requestBuilder =
         new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair());
@@ -749,11 +751,11 @@ public void testRegisterBlobSuccessResponse() throws Exception {
     CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
-    Mockito.when(statusLine.getStatusCode()).thenReturn(200);
-    Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine);
-    Mockito.when(httpResponse.getEntity()).thenReturn(httpEntity);
-    Mockito.when(httpEntity.getContent()).thenReturn(IOUtils.toInputStream(response));
-    Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
+    when(statusLine.getStatusCode()).thenReturn(200);
+    when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    when(httpResponse.getEntity()).thenReturn(httpEntity);
+    when(httpEntity.getContent()).thenReturn(IOUtils.toInputStream(response));
+    when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
 
     RequestBuilder requestBuilder =
         new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair());
@@ -827,16 +829,16 @@ public void testRegisterBlobsRetries() throws Exception {
     CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
-    Mockito.when(statusLine.getStatusCode()).thenReturn(200);
-    Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine);
-    Mockito.when(httpResponse.getEntity()).thenReturn(httpEntity);
-    Mockito.when(httpEntity.getContent())
+    when(statusLine.getStatusCode()).thenReturn(200);
+    when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    when(httpResponse.getEntity()).thenReturn(httpEntity);
+    when(httpEntity.getContent())
         .thenReturn(
             IOUtils.toInputStream(responseString),
             IOUtils.toInputStream(retryResponseString),
             IOUtils.toInputStream(retryResponseString),
             IOUtils.toInputStream(retryResponseString));
-    Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
+    when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
 
     RequestBuilder requestBuilder =
         Mockito.spy(
@@ -862,6 +864,92 @@ public void testRegisterBlobsRetries() throws Exception {
     Assert.assertFalse(channel2.isValid());
   }
 
+  @Test
+  public void testRegisterBlobChunkLimit() throws Exception {
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    RequestBuilder requestBuilder =
+        Mockito.spy(
+            new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair()));
+
+    SnowflakeStreamingIngestClientInternal<Object> client =
+        Mockito.spy(
+            new SnowflakeStreamingIngestClientInternal<>(
+                "client",
+                new SnowflakeURL("snowflake.dev.local:8082"),
+                null,
+                httpClient,
+                true,
+                requestBuilder,
+                null));
+
+    assertEquals(0, client.partitionBlobListForRegistrationRequest(new ArrayList<>()).size());
+    assertEquals(
+        1, client.partitionBlobListForRegistrationRequest(createTestBlobMetadata(1)).size());
+    assertEquals(
+        1, client.partitionBlobListForRegistrationRequest(createTestBlobMetadata(99)).size());
+    assertEquals(
+        1, client.partitionBlobListForRegistrationRequest(createTestBlobMetadata(100)).size());
+
+    assertEquals(
+        1, client.partitionBlobListForRegistrationRequest(createTestBlobMetadata(3, 95, 2)).size());
+    assertEquals(
+        2,
+        client.partitionBlobListForRegistrationRequest(createTestBlobMetadata(3, 95, 2, 1)).size());
+    assertEquals(
+        3,
+        client
+            .partitionBlobListForRegistrationRequest(createTestBlobMetadata(3, 95, 2, 1, 100))
+            .size());
+    assertEquals(
+        2, client.partitionBlobListForRegistrationRequest(createTestBlobMetadata(99, 2)).size());
+    assertEquals(
+        2,
+        client
+            .partitionBlobListForRegistrationRequest(createTestBlobMetadata(55, 44, 2, 98))
+            .size());
+    assertEquals(
+        3,
+        client
+            .partitionBlobListForRegistrationRequest(createTestBlobMetadata(55, 44, 2, 99))
+            .size());
+    assertEquals(
+        3,
+        client
+            .partitionBlobListForRegistrationRequest(createTestBlobMetadata(55, 44, 2, 99, 1))
+            .size());
+  }
+
+  /**
+   * Generate blob metadata with specified number of chunks per blob
+   *
+   * @param numbersOfChunks Array of chunk numbers per blob
+   * @return List of blob metadata
+   */
+  private List<BlobMetadata> createTestBlobMetadata(int... numbersOfChunks) {
+    List<BlobMetadata> result = new ArrayList<>();
+    for (int n : numbersOfChunks) {
+      List<ChunkMetadata> chunkMetadata = new ArrayList<>();
+      for (int i = 0; i < n; i++) {
+        ChunkMetadata chunk =
+            ChunkMetadata.builder()
+                .setOwningTableFromChannelContext(channel1.getChannelContext())
+                .setChunkStartOffset(0L)
+                .setChunkLength(1)
+                .setEncryptionKeyId(0L)
+                .setChunkMD5("")
+                .setEpInfo(new EpInfo())
+                .setChannelList(new ArrayList<>())
+                .setFirstInsertTimeInMs(0L)
+                .setLastInsertTimeInMs(0L)
+                .build();
+        chunkMetadata.add(chunk);
+      }
+
+      result.add(new BlobMetadata("", "", chunkMetadata, new BlobStats()));
+    }
+    return result;
+  }
+
   @Test
   public void testRegisterBlobsRetriesSucceeds() throws Exception {
     Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> testData = getRetryBlobMetadata();
@@ -945,13 +1033,13 @@ public void testRegisterBlobsRetriesSucceeds() throws Exception {
     CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
-    Mockito.when(statusLine.getStatusCode()).thenReturn(200);
-    Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine);
-    Mockito.when(httpResponse.getEntity()).thenReturn(httpEntity);
-    Mockito.when(httpEntity.getContent())
+    when(statusLine.getStatusCode()).thenReturn(200);
+    when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    when(httpResponse.getEntity()).thenReturn(httpEntity);
+    when(httpEntity.getContent())
         .thenReturn(
             IOUtils.toInputStream(responseString), IOUtils.toInputStream(retryResponseString));
-    Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
+    when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
 
     RequestBuilder requestBuilder =
         Mockito.spy(
@@ -1022,11 +1110,11 @@ public void testRegisterBlobResponseWithInvalidChannel() throws Exception {
     CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
-    Mockito.when(statusLine.getStatusCode()).thenReturn(200);
-    Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine);
-    Mockito.when(httpResponse.getEntity()).thenReturn(httpEntity);
-    Mockito.when(httpEntity.getContent()).thenReturn(IOUtils.toInputStream(response));
-    Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
+    when(statusLine.getStatusCode()).thenReturn(200);
+    when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    when(httpResponse.getEntity()).thenReturn(httpEntity);
+    when(httpEntity.getContent()).thenReturn(IOUtils.toInputStream(response));
+    when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
 
     RequestBuilder requestBuilder =
         new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair());
@@ -1146,7 +1234,7 @@ public void testCloseWithError() throws Exception {
 
     CompletableFuture<Void> future = new CompletableFuture<>();
     future.completeExceptionally(new Exception("Simulating Error"));
-    Mockito.when(client.flush(true)).thenReturn(future);
+    when(client.flush(true)).thenReturn(future);
 
     Assert.assertFalse(client.isClosed());
     try {
@@ -1249,11 +1337,11 @@ public void testGetLatestCommittedOffsetTokens() throws Exception {
     CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
     StatusLine statusLine = Mockito.mock(StatusLine.class);
     HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
-    Mockito.when(statusLine.getStatusCode()).thenReturn(200);
-    Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine);
-    Mockito.when(httpResponse.getEntity()).thenReturn(httpEntity);
-    Mockito.when(httpEntity.getContent()).thenReturn(IOUtils.toInputStream(responseString));
-    Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
+    when(statusLine.getStatusCode()).thenReturn(200);
+    when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    when(httpResponse.getEntity()).thenReturn(httpEntity);
+    when(httpEntity.getContent()).thenReturn(IOUtils.toInputStream(responseString));
+    when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
 
     RequestBuilder requestBuilder =
         Mockito.spy(

From 73713783a7f71aa78bc62d14a9225fa11af9995f Mon Sep 17 00:00:00 2001
From: Jiazhen Fan <52474868+sfc-gh-jfan@users.noreply.github.com>
Date: Wed, 18 Oct 2023 15:34:46 -0700
Subject: [PATCH 298/356] PRODSEC-3611 fix GHA parsing (#603)

---
 .github/workflows/snyk-issue.yml | 5 +++++
 .github/workflows/snyk-pr.yml    | 6 ++++++
 2 files changed, 11 insertions(+)

diff --git a/.github/workflows/snyk-issue.yml b/.github/workflows/snyk-issue.yml
index b586554dd..2a3f6226a 100644
--- a/.github/workflows/snyk-issue.yml
+++ b/.github/workflows/snyk-issue.yml
@@ -4,6 +4,11 @@ on:
   schedule:
     - cron: '* */12 * * *'
 
+permissions:
+  contents: read
+  issues: write
+  pull-requests: write
+
 concurrency: snyk-issue
 
 jobs:
diff --git a/.github/workflows/snyk-pr.yml b/.github/workflows/snyk-pr.yml
index 1ef32b622..4cb65c098 100644
--- a/.github/workflows/snyk-pr.yml
+++ b/.github/workflows/snyk-pr.yml
@@ -3,6 +3,12 @@ on:
   pull_request:
     branches:
       - master
+
+permissions:
+  contents: read
+  issues: write
+  pull-requests: write
+
 jobs:
   snyk:
     runs-on: ubuntu-latest

From 54aa2485eec43f4cce634efcbc99a0f1bb68ba69 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Thu, 19 Oct 2023 10:44:38 +0200
Subject: [PATCH 299/356] no-snow Sort pom.xml in format.sh (#604)

---
 format.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/format.sh b/format.sh
index 0cade08f5..cb58bd995 100755
--- a/format.sh
+++ b/format.sh
@@ -19,3 +19,6 @@ if ! command -v java > /dev/null; then
 fi
 echo "Running Google Java Format"
 find ./src -type f -name "*.java" -print0 | xargs -0 java -jar "${JAR_FILE}" --replace --set-exit-if-changed && echo "OK"
+
+echo "Sorting pom.xml"
+mvn com.github.ekryd.sortpom:sortpom-maven-plugin:sort

From cfb71c625bf155f2a48b087381e4931bbcbed33a Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Thu, 19 Oct 2023 11:29:01 +0200
Subject: [PATCH 300/356] SNOW-936900 Use upgraded version of snappy-java
 (#605)

The SDK is exporting an old version of its transitive dependency
snappy-java . This is visible in the new e2e jar project, which is
pulling in the snappy-java defined in hadoop-common and
not the version we define in dependencyManagement. This PR makes
snappy-java a direct dependency, so we can control the version.
---
 pom.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/pom.xml b/pom.xml
index fc4a2ab81..8a11ccb45 100644
--- a/pom.xml
+++ b/pom.xml
@@ -473,6 +473,12 @@
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.xerial.snappy</groupId>
+      <artifactId>snappy-java</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+
     <!-- JUnit so that we can make some basic unit tests -->
     <dependency>
       <groupId>junit</groupId>

From 4fd12a64549db6a631955d6e5becebd2f53d9f39 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Thu, 19 Oct 2023 21:23:40 +0200
Subject: [PATCH 301/356] SNOW-945927 SNOW-945928 Fix Snyk vulnerabilities
 (#641)

---
 e2e-jar-test/pom.xml | 6 ++++++
 pom.xml              | 5 +++++
 2 files changed, 11 insertions(+)

diff --git a/e2e-jar-test/pom.xml b/e2e-jar-test/pom.xml
index 9368308fc..1050d0967 100644
--- a/e2e-jar-test/pom.xml
+++ b/e2e-jar-test/pom.xml
@@ -36,6 +36,12 @@
         <version>3.13.30</version>
       </dependency>
 
+      <dependency>
+        <groupId>org.bouncycastle</groupId>
+        <artifactId>bc-fips</artifactId>
+        <version>1.0.2.4</version>
+      </dependency>
+
       <dependency>
         <groupId>org.slf4j</groupId>
         <artifactId>slf4j-simple</artifactId>
diff --git a/pom.xml b/pom.xml
index 8a11ccb45..9789e9e09 100644
--- a/pom.xml
+++ b/pom.xml
@@ -473,6 +473,11 @@
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <scope>runtime</scope>
+    </dependency>
     <dependency>
       <groupId>org.xerial.snappy</groupId>
       <artifactId>snappy-java</artifactId>

From 82f14b2838a25bf8fb0b9799b1ba11b27fc53d9e Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 19 Oct 2023 14:55:24 -0700
Subject: [PATCH 302/356] NO-SNOW: fix row index issue for ON_ERROR=SKIP_BATCH
 (#606)

We need to use a separate variable to store the index in the original batch instead of tempRowcount, added a test to verify the behavior
---
 .../streaming/internal/AbstractRowBuffer.java | 10 ++--
 .../streaming/internal/RowBufferTest.java     | 55 +++++++++++++++----
 2 files changed, 49 insertions(+), 16 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 3029e7a49..766a24763 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -223,13 +223,14 @@ public InsertValidationResponse insertRows(
       float rowsSizeInBytes = 0F;
       float tempRowsSizeInBytes = 0F;
       int tempRowCount = 0;
+      int rowIndex = 0;
       for (Map<String, Object> row : rows) {
         InsertValidationResponse.InsertError error =
-            new InsertValidationResponse.InsertError(row, tempRowCount);
+            new InsertValidationResponse.InsertError(row, rowIndex);
         try {
-          Set<String> inputColumnNames = verifyInputColumns(row, error, tempRowCount);
+          Set<String> inputColumnNames = verifyInputColumns(row, error, rowIndex);
           tempRowsSizeInBytes +=
-              addTempRow(row, tempRowCount, rowBuffer.tempStatsMap, inputColumnNames, tempRowCount);
+              addTempRow(row, tempRowCount, rowBuffer.tempStatsMap, inputColumnNames, rowIndex);
           tempRowCount++;
         } catch (SFException e) {
           error.setException(e);
@@ -239,8 +240,9 @@ public InsertValidationResponse insertRows(
           error.setException(new SFException(e, ErrorCode.INTERNAL_ERROR, e.getMessage()));
           response.addError(error);
         }
+        rowIndex++;
         checkBatchSizeEnforcedMaximum(tempRowsSizeInBytes);
-        if ((long) rowBuffer.bufferedRowCount + tempRowCount >= Integer.MAX_VALUE) {
+        if ((long) rowBuffer.bufferedRowCount + rowIndex >= Integer.MAX_VALUE) {
           throw new SFException(ErrorCode.INTERNAL_ERROR, "Row count reaches MAX value");
         }
       }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 269b6f28d..9ee1d29cb 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -14,7 +14,6 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.utils.Constants;
@@ -267,40 +266,54 @@ public void testInvalidPhysicalType() {
   public void testStringLength() {
     testStringLengthHelper(this.rowBufferOnErrorContinue);
     testStringLengthHelper(this.rowBufferOnErrorAbort);
+    testStringLengthHelper(this.rowBufferOnErrorSkipBatch);
   }
 
   @Test
   public void testRowIndexWithMultipleRowsWithError() {
+    testRowIndexWithMultipleRowsWithErrorHelper(this.rowBufferOnErrorContinue);
+    testRowIndexWithMultipleRowsWithErrorHelper(this.rowBufferOnErrorSkipBatch);
+  }
+
+  public void testRowIndexWithMultipleRowsWithErrorHelper(AbstractRowBuffer<?> rowBuffer) {
     List<Map<String, Object>> rows = new ArrayList<>();
     Map<String, Object> row = new HashMap<>();
 
-    // row with good data
+    // row with interleaved good and bad data
     row.put("colInt", 3);
     rows.add(row);
 
     row = new HashMap<>();
     row.put("colChar", "1111111111111111111111"); // too big
+    rows.add(row);
+
+    row = new HashMap<>();
+    row.put("colInt", 3);
+    rows.add(row);
 
-    // lets add a row with bad data
+    row = new HashMap<>();
+    row.put("colChar", "1111111111111111111111"); // too big
     rows.add(row);
 
-    InsertValidationResponse response = this.rowBufferOnErrorContinue.insertRows(rows, null);
+    InsertValidationResponse response = rowBuffer.insertRows(rows, null);
     Assert.assertTrue(response.hasErrors());
 
-    Assert.assertEquals(1, response.getErrorRowCount());
+    Assert.assertEquals(2, response.getErrorRowCount());
 
     // second row out of the rows we sent was having bad data.
     // so InsertError corresponds to second row.
     Assert.assertEquals(1, response.getInsertErrors().get(0).getRowIndex());
+    Assert.assertEquals(3, response.getInsertErrors().get(1).getRowIndex());
 
-    Assert.assertTrue(response.getInsertErrors().get(0).getException() != null);
+    Assert.assertNotNull(response.getInsertErrors().get(0).getException());
+    Assert.assertNotNull(response.getInsertErrors().get(1).getException());
 
-    Assert.assertTrue(
-        Objects.equals(
-            response.getInsertErrors().get(0).getException().getVendorCode(),
-            ErrorCode.INVALID_VALUE_ROW.getMessageCode()));
-
-    Assert.assertEquals(1, response.getInsertErrors().get(0).getRowIndex());
+    Assert.assertEquals(
+        response.getInsertErrors().get(0).getException().getVendorCode(),
+        ErrorCode.INVALID_VALUE_ROW.getMessageCode());
+    Assert.assertEquals(
+        response.getInsertErrors().get(1).getException().getVendorCode(),
+        ErrorCode.INVALID_VALUE_ROW.getMessageCode());
 
     Assert.assertTrue(
         response
@@ -313,6 +326,17 @@ public void testRowIndexWithMultipleRowsWithError() {
                     + " Value cannot be ingested into Snowflake column COLCHAR of type STRING,"
                     + " rowIndex:1, reason: String too long: length=22 characters maxLength=11"
                     + " characters"));
+    Assert.assertTrue(
+        response
+            .getInsertErrors()
+            .get(1)
+            .getException()
+            .getMessage()
+            .equalsIgnoreCase(
+                "The given row cannot be converted to the internal format due to invalid value:"
+                    + " Value cannot be ingested into Snowflake column COLCHAR of type STRING,"
+                    + " rowIndex:3, reason: String too long: length=22 characters maxLength=11"
+                    + " characters"));
   }
 
   private void testStringLengthHelper(AbstractRowBuffer<?> rowBuffer) {
@@ -357,6 +381,7 @@ private void testStringLengthHelper(AbstractRowBuffer<?> rowBuffer) {
   public void testInsertRow() {
     testInsertRowHelper(this.rowBufferOnErrorContinue);
     testInsertRowHelper(this.rowBufferOnErrorAbort);
+    testInsertRowHelper(this.rowBufferOnErrorSkipBatch);
   }
 
   private void testInsertRowHelper(AbstractRowBuffer<?> rowBuffer) {
@@ -377,6 +402,7 @@ private void testInsertRowHelper(AbstractRowBuffer<?> rowBuffer) {
   public void testNullInsertRow() {
     testInsertNullRowHelper(this.rowBufferOnErrorContinue);
     testInsertNullRowHelper(this.rowBufferOnErrorAbort);
+    testInsertNullRowHelper(this.rowBufferOnErrorSkipBatch);
   }
 
   private void testInsertNullRowHelper(AbstractRowBuffer<?> rowBuffer) {
@@ -397,6 +423,7 @@ private void testInsertNullRowHelper(AbstractRowBuffer<?> rowBuffer) {
   public void testInsertRows() {
     testInsertRowsHelper(this.rowBufferOnErrorContinue);
     testInsertRowsHelper(this.rowBufferOnErrorAbort);
+    testInsertRowsHelper(this.rowBufferOnErrorSkipBatch);
   }
 
   private void testInsertRowsHelper(AbstractRowBuffer<?> rowBuffer) {
@@ -426,6 +453,7 @@ private void testInsertRowsHelper(AbstractRowBuffer<?> rowBuffer) {
   public void testFlush() {
     testFlushHelper(this.rowBufferOnErrorAbort);
     testFlushHelper(this.rowBufferOnErrorContinue);
+    testFlushHelper(this.rowBufferOnErrorSkipBatch);
   }
 
   private void testFlushHelper(AbstractRowBuffer<?> rowBuffer) {
@@ -598,6 +626,7 @@ public void testInvalidEPInfo() {
   public void testE2E() {
     testE2EHelper(this.rowBufferOnErrorAbort);
     testE2EHelper(this.rowBufferOnErrorContinue);
+    testE2EHelper(this.rowBufferOnErrorSkipBatch);
   }
 
   private void testE2EHelper(AbstractRowBuffer<?> rowBuffer) {
@@ -626,6 +655,7 @@ private void testE2EHelper(AbstractRowBuffer<?> rowBuffer) {
   public void testE2ETimestampErrors() {
     testE2ETimestampErrorsHelper(this.rowBufferOnErrorAbort);
     testE2ETimestampErrorsHelper(this.rowBufferOnErrorContinue);
+    testE2ETimestampErrorsHelper(this.rowBufferOnErrorSkipBatch);
   }
 
   private void testE2ETimestampErrorsHelper(AbstractRowBuffer<?> innerBuffer) {
@@ -663,6 +693,7 @@ private void testE2ETimestampErrorsHelper(AbstractRowBuffer<?> innerBuffer) {
   public void testStatsE2E() {
     testStatsE2EHelper(this.rowBufferOnErrorAbort);
     testStatsE2EHelper(this.rowBufferOnErrorContinue);
+    testStatsE2EHelper(this.rowBufferOnErrorSkipBatch);
   }
 
   private void testStatsE2EHelper(AbstractRowBuffer<?> rowBuffer) {

From d7cd5a04948032d8cc4f498a4f43590c5cc35777 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Mon, 23 Oct 2023 22:57:22 -0700
Subject: [PATCH 303/356] V2.0.4 release (#643)

v2.0.4 release, all the additional changes are done by sorting the pom file, looks like this is added as part of #604
---
 e2e-jar-test/pom.xml                          | 104 +++++++++---------
 pom.xml                                       |  26 ++---
 .../ingest/connection/RequestBuilder.java     |   3 +-
 3 files changed, 67 insertions(+), 66 deletions(-)

diff --git a/e2e-jar-test/pom.xml b/e2e-jar-test/pom.xml
index 1050d0967..9a140aaeb 100644
--- a/e2e-jar-test/pom.xml
+++ b/e2e-jar-test/pom.xml
@@ -1,65 +1,65 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
 
-  <groupId>net.snowflake.snowflake-ingest-java-e2e-jar-test</groupId>
-  <artifactId>parent</artifactId>
-  <version>1.0-SNAPSHOT</version>
-  <packaging>pom</packaging>
+    <groupId>net.snowflake.snowflake-ingest-java-e2e-jar-test</groupId>
+    <artifactId>parent</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <packaging>pom</packaging>
 
-  <name>snowflake-ingest-sdk-e2e-test</name>
+    <name>snowflake-ingest-sdk-e2e-test</name>
 
-  <modules>
-    <module>standard</module>
-    <module>fips</module>
-    <module>core</module>
-  </modules>
+    <modules>
+        <module>standard</module>
+        <module>fips</module>
+        <module>core</module>
+    </modules>
 
-  <dependencyManagement>
+    <dependencyManagement>
 
-    <dependencies>
-      <dependency>
-        <groupId>net.snowflake.snowflake-ingest-java-e2e-jar-test</groupId>
-        <artifactId>core</artifactId>
-        <version>${project.version}</version>
-      </dependency>
+        <dependencies>
+            <dependency>
+                <groupId>net.snowflake.snowflake-ingest-java-e2e-jar-test</groupId>
+                <artifactId>core</artifactId>
+                <version>${project.version}</version>
+            </dependency>
 
-      <dependency>
-        <groupId>net.snowflake</groupId>
-        <artifactId>snowflake-ingest-sdk</artifactId>
-        <version>2.0.4-SNAPSHOT</version>
-      </dependency>
+            <dependency>
+                <groupId>net.snowflake</groupId>
+                <artifactId>snowflake-ingest-sdk</artifactId>
+                <version>2.0.4</version>
+            </dependency>
 
-      <dependency>
-        <groupId>net.snowflake</groupId>
-        <artifactId>snowflake-jdbc-fips</artifactId>
-        <version>3.13.30</version>
-      </dependency>
+            <dependency>
+                <groupId>net.snowflake</groupId>
+                <artifactId>snowflake-jdbc-fips</artifactId>
+                <version>3.13.30</version>
+            </dependency>
 
-      <dependency>
-        <groupId>org.bouncycastle</groupId>
-        <artifactId>bc-fips</artifactId>
-        <version>1.0.2.4</version>
-      </dependency>
+            <dependency>
+                <groupId>org.bouncycastle</groupId>
+                <artifactId>bc-fips</artifactId>
+                <version>1.0.2.4</version>
+            </dependency>
 
-      <dependency>
-        <groupId>org.slf4j</groupId>
-        <artifactId>slf4j-simple</artifactId>
-        <version>1.7.36</version>
-      </dependency>
+            <dependency>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-simple</artifactId>
+                <version>1.7.36</version>
+            </dependency>
 
-      <dependency>
-        <groupId>com.fasterxml.jackson.core</groupId>
-        <artifactId>jackson-databind</artifactId>
-        <version>2.14.0</version>
-      </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-databind</artifactId>
+                <version>2.14.0</version>
+            </dependency>
 
-      <dependency>
-        <groupId>junit</groupId>
-        <artifactId>junit</artifactId>
-        <version>4.13.2</version>
-        <scope>test</scope>
-      </dependency>
-    </dependencies>
-  </dependencyManagement>
+            <dependency>
+                <groupId>junit</groupId>
+                <artifactId>junit</artifactId>
+                <version>4.13.2</version>
+                <scope>test</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
 </project>
diff --git a/pom.xml b/pom.xml
index 9789e9e09..90f5ccbb5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.0.4-SNAPSHOT</version>
+  <version>2.0.4</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
@@ -779,9 +779,9 @@
         <configuration>
           <errorRemedy>failFast</errorRemedy>
           <!--
-          The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please
-          check your dependencies", verify the conditions of the license and add the reference to it here.
-          -->
+                    The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please
+                    check your dependencies", verify the conditions of the license and add the reference to it here.
+                    -->
           <includedLicenses>
             <includedLicense>Apache License 2.0</includedLicense>
             <includedLicense>BSD 2-Clause License</includedLicense>
@@ -899,9 +899,9 @@
       <build>
         <plugins>
           <!--
-          Copy all project dependencies to target/dependency-jars. License processing Python script will look here for
-          license files of SDK dependencies.
-          -->
+                    Copy all project dependencies to target/dependency-jars. License processing Python script will look here for
+                    license files of SDK dependencies.
+                    -->
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-dependency-plugin</artifactId>
@@ -923,9 +923,9 @@
           </plugin>
 
           <!--
-          Compile the list of SDK dependencies in 'compile' and 'runtime' scopes.
-          This list is an entry point for the license processing python script.
-          -->
+                    Compile the list of SDK dependencies in 'compile' and 'runtime' scopes.
+                    This list is an entry point for the license processing python script.
+                    -->
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-dependency-plugin</artifactId>
@@ -1108,9 +1108,9 @@
             </executions>
           </plugin>
           <!--
-          Plugin executes license processing Python script, which copies third party license files into the directory
-          target/generated-licenses-info/META-INF/third-party-licenses, which is then included in the shaded JAR. 
-          -->
+                    Plugin executes license processing Python script, which copies third party license files into the directory
+                    target/generated-licenses-info/META-INF/third-party-licenses, which is then included in the shaded JAR.
+                    -->
           <plugin>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>exec-maven-plugin</artifactId>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 6936bc52a..77443e7b8 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.0.4-SNAPSHOT";
+  public static final String DEFAULT_VERSION = "2.0.4";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 
@@ -351,6 +351,7 @@ private static String getDefaultUserAgent() {
   private static String buildCustomUserAgent(String additionalUserAgentInfo) {
     return USER_AGENT.trim() + " " + additionalUserAgentInfo;
   }
+
   /** A simple POJO for generating our POST body to the insert endpoint */
   private static class IngestRequest {
     // the list of files we're loading

From 9bc5859da49b65be72f32db3b8393ea95dbdefc2 Mon Sep 17 00:00:00 2001
From: Alkin Sen <120425561+sfc-gh-asen@users.noreply.github.com>
Date: Wed, 25 Oct 2023 16:01:36 -0700
Subject: [PATCH 304/356] SNOW-949967 Add an optional offset token parameter
 for openChannel (#645)

---
 .../ingest/streaming/OpenChannelRequest.java  | 22 ++++++++++++
 ...nowflakeStreamingIngestClientInternal.java |  3 ++
 .../SnowflakeStreamingIngestChannelTest.java  | 19 ++++++++++
 .../streaming/internal/StreamingIngestIT.java | 35 +++++++++++++++++++
 4 files changed, 79 insertions(+)

diff --git a/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java b/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java
index f48288330..d30472b63 100644
--- a/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java
+++ b/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java
@@ -40,6 +40,9 @@ public enum OnErrorOption {
   // Default timezone for TIMESTAMP_LTZ and TIMESTAMP_TZ columns
   private final ZoneId defaultTimezone;
 
+  private final String offsetToken;
+  private final boolean isOffsetTokenProvided;
+
   public static OpenChannelRequestBuilder builder(String channelName) {
     return new OpenChannelRequestBuilder(channelName);
   }
@@ -53,6 +56,9 @@ public static class OpenChannelRequestBuilder {
     private OnErrorOption onErrorOption;
     private ZoneId defaultTimezone;
 
+    private String offsetToken;
+    private boolean isOffsetTokenProvided = false;
+
     public OpenChannelRequestBuilder(String channelName) {
       this.channelName = channelName;
       this.defaultTimezone = DEFAULT_DEFAULT_TIMEZONE;
@@ -83,6 +89,12 @@ public OpenChannelRequestBuilder setDefaultTimezone(ZoneId defaultTimezone) {
       return this;
     }
 
+    public OpenChannelRequestBuilder setOffsetToken(String offsetToken){
+      this.offsetToken = offsetToken;
+      this.isOffsetTokenProvided = true;
+      return  this;
+    }
+
     public OpenChannelRequest build() {
       return new OpenChannelRequest(this);
     }
@@ -102,6 +114,8 @@ private OpenChannelRequest(OpenChannelRequestBuilder builder) {
     this.tableName = builder.tableName;
     this.onErrorOption = builder.onErrorOption;
     this.defaultTimezone = builder.defaultTimezone;
+    this.offsetToken = builder.offsetToken;
+    this.isOffsetTokenProvided = builder.isOffsetTokenProvided;
   }
 
   public String getDBName() {
@@ -131,4 +145,12 @@ public String getFullyQualifiedTableName() {
   public OnErrorOption getOnErrorOption() {
     return this.onErrorOption;
   }
+
+  public String getOffsetToken() {
+    return this.offsetToken;
+  }
+
+  public boolean isOffsetTokenProvided() {
+    return this.isOffsetTokenProvided;
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 27ff9407e..a52149331 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -304,6 +304,9 @@ public SnowflakeStreamingIngestChannelInternal<?> openChannel(OpenChannelRequest
       payload.put("schema", request.getSchemaName());
       payload.put("write_mode", Constants.WriteMode.CLOUD_STORAGE.name());
       payload.put("role", this.role);
+      if (request.isOffsetTokenProvided()){
+        payload.put("offset_token", request.getOffsetToken());
+      }
 
       OpenChannelResponse response =
           executeWithRetries(
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index a1e26b63d..037028086 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -263,6 +263,25 @@ public void testOpenChannelRequestCreationSuccess() {
 
     Assert.assertEquals(
         "STREAMINGINGEST_TEST.PUBLIC.T_STREAMINGINGEST", request.getFullyQualifiedTableName());
+    Assert.assertFalse(request.isOffsetTokenProvided());
+  }
+
+
+  @Test
+  public void testOpenChannelRequesCreationtWithOffsetToken() {
+    OpenChannelRequest request =
+            OpenChannelRequest.builder("CHANNEL")
+                    .setDBName("STREAMINGINGEST_TEST")
+                    .setSchemaName("PUBLIC")
+                    .setTableName("T_STREAMINGINGEST")
+                    .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+                    .setOffsetToken("TEST_TOKEN")
+                    .build();
+
+    Assert.assertEquals(
+            "STREAMINGINGEST_TEST.PUBLIC.T_STREAMINGINGEST", request.getFullyQualifiedTableName());
+    Assert.assertEquals("TEST_TOKEN", request.getOffsetToken());
+    Assert.assertTrue(request.isOffsetTokenProvided());
   }
 
   @Test
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 95cc5c699..eadbe81ee 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -532,6 +532,41 @@ public void testMultiColumnIngest() throws Exception {
     Assert.fail("Row sequencer not updated before timeout");
   }
 
+  @Test
+  public void testOpenChannelOffsetToken() throws Exception {
+    String tableName = "offsetTokenTest";
+    jdbcConnection
+            .createStatement()
+            .execute(
+                    String.format(
+                            "create or replace table %s (s text);",
+                            tableName));
+    OpenChannelRequest request1 =
+            OpenChannelRequest.builder("TEST_CHANNEL")
+                    .setDBName(testDb)
+                    .setSchemaName(TEST_SCHEMA)
+                    .setTableName(tableName)
+                    .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+                    .setOffsetToken("TEST_OFFSET")
+                    .build();
+
+    // Open a streaming ingest channel from the given client
+    SnowflakeStreamingIngestChannel channel1 = client.openChannel(request1);
+
+    // Close the channel after insertion
+    channel1.close().get();
+
+    for (int i = 1; i < 15; i++) {
+      if (channel1.getLatestCommittedOffsetToken() != null
+              && channel1.getLatestCommittedOffsetToken().equals("TEST_OFFSET")) {
+        return;
+      } else {
+        Thread.sleep(2000);
+      }
+    }
+    Assert.fail("Row sequencer not updated before timeout");
+  }
+
   @Test
   public void testNullableColumns() throws Exception {
     String multiTableName = "multi_column";

From 5ef99b8da699dc6ccdaff9e4dd3b7a25e4878894 Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Tue, 31 Oct 2023 12:54:51 +0100
Subject: [PATCH 305/356] SNOW-672156 support specifying compression algorithm
 to be used for BDEC Parquet files. GZIP is the only allowed value for now.
 (#579)

---
 .../internal/ClientBufferParameters.java      | 23 ++++++++--
 .../streaming/internal/ParquetFlusher.java    | 16 ++++++-
 .../streaming/internal/ParquetRowBuffer.java  |  6 ++-
 .../net/snowflake/ingest/utils/Constants.java | 24 +++++++++++
 .../ingest/utils/ParameterProvider.java       | 24 ++++++++++-
 .../parquet/hadoop/BdecParquetWriter.java     |  7 ++--
 .../internal/ParameterProviderTest.java       | 42 +++++++++++++++++++
 .../streaming/internal/RowBufferTest.java     |  3 +-
 8 files changed, 133 insertions(+), 12 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ClientBufferParameters.java b/src/main/java/net/snowflake/ingest/streaming/internal/ClientBufferParameters.java
index dffd824c8..278d4abea 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ClientBufferParameters.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ClientBufferParameters.java
@@ -4,6 +4,7 @@
 
 package net.snowflake.ingest.streaming.internal;
 
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ParameterProvider;
 
 /** Channel's buffer relevant parameters that are set at the owning client level. */
@@ -15,6 +16,8 @@ public class ClientBufferParameters {
 
   private long maxAllowedRowSizeInBytes;
 
+  private Constants.BdecParquetCompression bdecParquetCompression;
+
   /**
    * Private constructor used for test methods
    *
@@ -26,10 +29,12 @@ public class ClientBufferParameters {
   private ClientBufferParameters(
       boolean enableParquetInternalBuffering,
       long maxChunkSizeInBytes,
-      long maxAllowedRowSizeInBytes) {
+      long maxAllowedRowSizeInBytes,
+      Constants.BdecParquetCompression bdecParquetCompression) {
     this.enableParquetInternalBuffering = enableParquetInternalBuffering;
     this.maxChunkSizeInBytes = maxChunkSizeInBytes;
     this.maxAllowedRowSizeInBytes = maxAllowedRowSizeInBytes;
+    this.bdecParquetCompression = bdecParquetCompression;
   }
 
   /** @param clientInternal reference to the client object where the relevant parameters are set */
@@ -46,6 +51,10 @@ public ClientBufferParameters(SnowflakeStreamingIngestClientInternal clientInter
         clientInternal != null
             ? clientInternal.getParameterProvider().getMaxAllowedRowSizeInBytes()
             : ParameterProvider.MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT;
+    this.bdecParquetCompression =
+        clientInternal != null
+            ? clientInternal.getParameterProvider().getBdecParquetCompressionAlgorithm()
+            : ParameterProvider.BDEC_PARQUET_COMPRESSION_ALGORITHM_DEFAULT;
   }
 
   /**
@@ -58,9 +67,13 @@ public ClientBufferParameters(SnowflakeStreamingIngestClientInternal clientInter
   public static ClientBufferParameters test_createClientBufferParameters(
       boolean enableParquetInternalBuffering,
       long maxChunkSizeInBytes,
-      long maxAllowedRowSizeInBytes) {
+      long maxAllowedRowSizeInBytes,
+      Constants.BdecParquetCompression bdecParquetCompression) {
     return new ClientBufferParameters(
-        enableParquetInternalBuffering, maxChunkSizeInBytes, maxAllowedRowSizeInBytes);
+        enableParquetInternalBuffering,
+        maxChunkSizeInBytes,
+        maxAllowedRowSizeInBytes,
+        bdecParquetCompression);
   }
 
   public boolean getEnableParquetInternalBuffering() {
@@ -74,4 +87,8 @@ public long getMaxChunkSizeInBytes() {
   public long getMaxAllowedRowSizeInBytes() {
     return maxAllowedRowSizeInBytes;
   }
+
+  public Constants.BdecParquetCompression getBdecParquetCompression() {
+    return bdecParquetCompression;
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
index 87c30207d..e81dbc0eb 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
@@ -9,6 +9,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.Logging;
 import net.snowflake.ingest.utils.Pair;
@@ -27,15 +28,21 @@ public class ParquetFlusher implements Flusher<ParquetChunkData> {
   private final boolean enableParquetInternalBuffering;
   private final long maxChunkSizeInBytes;
 
+  private final Constants.BdecParquetCompression bdecParquetCompression;
+
   /**
    * Construct parquet flusher from its schema and set flag that indicates whether Parquet memory
    * optimization is enabled, i.e. rows will be buffered in internal Parquet buffer.
    */
   public ParquetFlusher(
-      MessageType schema, boolean enableParquetInternalBuffering, long maxChunkSizeInBytes) {
+      MessageType schema,
+      boolean enableParquetInternalBuffering,
+      long maxChunkSizeInBytes,
+      Constants.BdecParquetCompression bdecParquetCompression) {
     this.schema = schema;
     this.enableParquetInternalBuffering = enableParquetInternalBuffering;
     this.maxChunkSizeInBytes = maxChunkSizeInBytes;
+    this.bdecParquetCompression = bdecParquetCompression;
   }
 
   @Override
@@ -198,7 +205,12 @@ private SerializationResult serializeFromJavaObjects(
     Map<String, String> metadata = channelsDataPerTable.get(0).getVectors().metadata;
     parquetWriter =
         new BdecParquetWriter(
-            mergedData, schema, metadata, firstChannelFullyQualifiedTableName, maxChunkSizeInBytes);
+            mergedData,
+            schema,
+            metadata,
+            firstChannelFullyQualifiedTableName,
+            maxChunkSizeInBytes,
+            bdecParquetCompression);
     rows.forEach(parquetWriter::writeRow);
     parquetWriter.close();
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 6fb7b8027..602e34400 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -122,7 +122,8 @@ private void createFileWriter() {
                 schema,
                 metadata,
                 channelName,
-                clientBufferParameters.getMaxChunkSizeInBytes());
+                clientBufferParameters.getMaxChunkSizeInBytes(),
+                clientBufferParameters.getBdecParquetCompression());
       } else {
         this.bdecParquetWriter = null;
       }
@@ -323,7 +324,8 @@ public Flusher<ParquetChunkData> createFlusher() {
     return new ParquetFlusher(
         schema,
         clientBufferParameters.getEnableParquetInternalBuffering(),
-        clientBufferParameters.getMaxChunkSizeInBytes());
+        clientBufferParameters.getMaxChunkSizeInBytes(),
+        clientBufferParameters.getBdecParquetCompression());
   }
 
   private static class ParquetColumn {
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index 09814235b..a22bc9f21 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -5,6 +5,7 @@
 package net.snowflake.ingest.utils;
 
 import java.util.Arrays;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
 
 /** Contains all the constants needed for Streaming Ingest */
 public class Constants {
@@ -113,6 +114,29 @@ public static BdecVersion fromInt(int val) {
     }
   }
 
+  /**
+   * Compression algorithm supported by BDEC Parquet Writer. It is a wrapper around Parquet's lib
+   * CompressionCodecName, but we want to control and allow only specific values of that.
+   */
+  public enum BdecParquetCompression {
+    GZIP;
+
+    public CompressionCodecName getCompressionCodec() {
+      return CompressionCodecName.fromConf(this.name());
+    }
+
+    public static BdecParquetCompression fromName(String name) {
+      for (BdecParquetCompression e : BdecParquetCompression.values()) {
+        if (e.name().toLowerCase().equals(name.toLowerCase())) {
+          return e;
+        }
+      }
+      throw new IllegalArgumentException(
+          String.format(
+              "Unsupported BDEC_PARQUET_COMPRESSION_ALGORITHM = '%s', allowed values are %s",
+              name, Arrays.asList(BdecParquetCompression.values())));
+    }
+  }
   // Parameters
   public static final boolean DISABLE_BACKGROUND_FLUSH = false;
   public static final boolean COMPRESS_BLOB_TWICE = false;
diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
index 3a2221697..82dd1164b 100644
--- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
+++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -38,6 +38,9 @@ public class ParameterProvider {
 
   public static final String MAX_CLIENT_LAG_ENABLED = "MAX_CLIENT_LAG_ENABLED".toLowerCase();
 
+  public static final String BDEC_PARQUET_COMPRESSION_ALGORITHM =
+      "BDEC_PARQUET_COMPRESSION_ALGORITHM".toLowerCase();
+
   // Default values
   public static final long BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT = 1000;
   public static final long BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_DEFAULT = 100;
@@ -63,6 +66,9 @@ public class ParameterProvider {
   public static final long MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT = 64 * 1024 * 1024; // 64 MB
   public static final int MAX_CHUNKS_IN_BLOB_AND_REGISTRATION_REQUEST_DEFAULT = 100;
 
+  public static final Constants.BdecParquetCompression BDEC_PARQUET_COMPRESSION_ALGORITHM_DEFAULT =
+      Constants.BdecParquetCompression.GZIP;
+
   /* Parameter that enables using internal Parquet buffers for buffering of rows before serializing.
   It reduces memory consumption compared to using Java Objects for buffering.*/
   public static final boolean ENABLE_PARQUET_INTERNAL_BUFFERING_DEFAULT = false;
@@ -178,6 +184,12 @@ private void setParameterMap(Map<String, Object> parameterOverrides, Properties
         MAX_CHUNKS_IN_BLOB_AND_REGISTRATION_REQUEST_DEFAULT,
         parameterOverrides,
         props);
+
+    this.updateValue(
+        BDEC_PARQUET_COMPRESSION_ALGORITHM,
+        BDEC_PARQUET_COMPRESSION_ALGORITHM_DEFAULT,
+        parameterOverrides,
+        props);
   }
 
   /** @return Longest interval in milliseconds between buffer flushes */
@@ -377,7 +389,6 @@ public long getMaxChunkSizeInBytes() {
     return (val instanceof String) ? Long.parseLong(val.toString()) : (long) val;
   }
 
-  /** @return The max allow row size (in bytes) */
   public long getMaxAllowedRowSizeInBytes() {
     Object val =
         this.parameterMap.getOrDefault(
@@ -397,6 +408,17 @@ public int getMaxChunksInBlobAndRegistrationRequest() {
     return (val instanceof String) ? Integer.parseInt(val.toString()) : (int) val;
   }
 
+  /** @return BDEC compression algorithm */
+  public Constants.BdecParquetCompression getBdecParquetCompressionAlgorithm() {
+    Object val =
+            this.parameterMap.getOrDefault(
+                    BDEC_PARQUET_COMPRESSION_ALGORITHM, BDEC_PARQUET_COMPRESSION_ALGORITHM_DEFAULT);
+    if (val instanceof Constants.BdecParquetCompression) {
+      return (Constants.BdecParquetCompression) val;
+    }
+    return Constants.BdecParquetCompression.fromName((String) val);
+  }
+
   @Override
   public String toString() {
     return "ParameterProvider{" + "parameterMap=" + parameterMap + '}';
diff --git a/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java b/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java
index b2442d3dc..8b71cfd0e 100644
--- a/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java
+++ b/src/main/java/org/apache/parquet/hadoop/BdecParquetWriter.java
@@ -17,7 +17,6 @@
 import org.apache.parquet.column.values.factory.DefaultV1ValuesWriterFactory;
 import org.apache.parquet.crypto.FileEncryptionProperties;
 import org.apache.parquet.hadoop.api.WriteSupport;
-import org.apache.parquet.hadoop.metadata.CompressionCodecName;
 import org.apache.parquet.io.DelegatingPositionOutputStream;
 import org.apache.parquet.io.OutputFile;
 import org.apache.parquet.io.ParquetEncodingException;
@@ -51,7 +50,8 @@ public BdecParquetWriter(
       MessageType schema,
       Map<String, String> extraMetaData,
       String channelName,
-      long maxChunkSizeInBytes)
+      long maxChunkSizeInBytes,
+      Constants.BdecParquetCompression bdecParquetCompression)
       throws IOException {
     OutputFile file = new ByteArrayOutputFile(stream, maxChunkSizeInBytes);
     ParquetProperties encodingProps = createParquetProperties();
@@ -86,7 +86,8 @@ public BdecParquetWriter(
     */
     codecFactory = new CodecFactory(conf, ParquetWriter.DEFAULT_PAGE_SIZE);
     @SuppressWarnings("deprecation") // Parquet does not support the new one now
-    CodecFactory.BytesCompressor compressor = codecFactory.getCompressor(CompressionCodecName.GZIP);
+    CodecFactory.BytesCompressor compressor =
+        codecFactory.getCompressor(bdecParquetCompression.getCompressionCodec());
     writer =
         new InternalParquetRecordWriter<>(
             fileWriter,
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
index 7838763de..8499fc985 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
@@ -1,8 +1,11 @@
 package net.snowflake.ingest.streaming.internal;
 
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ParameterProvider;
 import org.junit.Assert;
 import org.junit.Test;
@@ -20,6 +23,7 @@ private Map<String, Object> getStartingParameterMap() {
     parameterMap.put(ParameterProvider.BLOB_UPLOAD_MAX_RETRY_COUNT, 100);
     parameterMap.put(ParameterProvider.MAX_MEMORY_LIMIT_IN_BYTES, 1000L);
     parameterMap.put(ParameterProvider.MAX_CHANNEL_SIZE_IN_BYTES, 1000000L);
+    parameterMap.put(ParameterProvider.BDEC_PARQUET_COMPRESSION_ALGORITHM, "gzip");
     return parameterMap;
   }
 
@@ -39,6 +43,9 @@ public void withValuesSet() {
     Assert.assertEquals(100, parameterProvider.getBlobUploadMaxRetryCount());
     Assert.assertEquals(1000L, parameterProvider.getMaxMemoryLimitInBytes());
     Assert.assertEquals(1000000L, parameterProvider.getMaxChannelSizeInBytes());
+    Assert.assertEquals(
+        Constants.BdecParquetCompression.GZIP,
+        parameterProvider.getBdecParquetCompressionAlgorithm());
   }
 
   @Test
@@ -130,6 +137,9 @@ public void withDefaultValues() {
     Assert.assertEquals(
         ParameterProvider.MAX_CHANNEL_SIZE_IN_BYTES_DEFAULT,
         parameterProvider.getMaxChannelSizeInBytes());
+    Assert.assertEquals(
+        ParameterProvider.BDEC_PARQUET_COMPRESSION_ALGORITHM_DEFAULT,
+        parameterProvider.getBdecParquetCompressionAlgorithm());
   }
 
   @Test
@@ -281,4 +291,36 @@ public void testMaxChunksInBlobAndRegistrationRequest() {
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
     Assert.assertEquals(1, parameterProvider.getMaxChunksInBlobAndRegistrationRequest());
   }
+
+  @Test
+  public void testValidCompressionAlgorithmsAndWithUppercaseLowerCase() {
+    List<String> gzipValues = Arrays.asList("GZIP", "gzip", "Gzip", "gZip");
+    gzipValues.forEach(
+        v -> {
+          Properties prop = new Properties();
+          Map<String, Object> parameterMap = getStartingParameterMap();
+          parameterMap.put(ParameterProvider.BDEC_PARQUET_COMPRESSION_ALGORITHM, v);
+          ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+          Assert.assertEquals(
+              Constants.BdecParquetCompression.GZIP,
+              parameterProvider.getBdecParquetCompressionAlgorithm());
+        });
+  }
+
+  @Test
+  public void testInvalidCompressionAlgorithm() {
+    Properties prop = new Properties();
+    Map<String, Object> parameterMap = getStartingParameterMap();
+    parameterMap.put(ParameterProvider.BDEC_PARQUET_COMPRESSION_ALGORITHM, "invalid_comp");
+    ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+    try {
+      parameterProvider.getBdecParquetCompressionAlgorithm();
+      Assert.fail("Should not have succeeded");
+    } catch (IllegalArgumentException e) {
+      Assert.assertEquals(
+          "Unsupported BDEC_PARQUET_COMPRESSION_ALGORITHM = 'invalid_comp', allowed values are"
+              + " [GZIP]",
+          e.getMessage());
+    }
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 9ee1d29cb..52d8eed2e 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -120,7 +120,8 @@ private AbstractRowBuffer<?> createTestBuffer(OpenChannelRequest.OnErrorOption o
         ClientBufferParameters.test_createClientBufferParameters(
             enableParquetMemoryOptimization,
             MAX_CHUNK_SIZE_IN_BYTES_DEFAULT,
-            MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT));
+            MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT,
+            Constants.BdecParquetCompression.GZIP));
   }
 
   @Test

From 2a3dd58cf48c49fa76a81e550b96fbef77dbfa44 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 2 Nov 2023 15:51:04 -0700
Subject: [PATCH 306/356] SNOW-957347: Disable Snowpipe flaky tests (#647)

SNOW-957347: Disable Snowpipe flaky tests
---
 src/test/java/net/snowflake/ingest/SimpleIngestIT.java | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
index b1eafc93e..ee73a2046 100644
--- a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
@@ -28,6 +28,7 @@
 import net.snowflake.ingest.utils.StagedFileWrapper;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
 /** Example ingest sdk integration test */
@@ -123,6 +124,7 @@ public void afterAll() {
 
   /** ingest test example ingest a simple file and check load history. */
   @Test
+  @Ignore // SNOW-957347: re-enable after fix
   public void testSimpleIngest() throws Exception {
     // put
     TestUtils.executeQuery("put file://" + testFilePath + " @" + stageName);
@@ -176,7 +178,8 @@ public void testSimpleIngestWithPattern() throws Exception {
   }
 
   private void getHistoryAndAssertLoad(SimpleIngestManager manager, String test_file_name_2)
-      throws InterruptedException, java.util.concurrent.ExecutionException,
+      throws InterruptedException,
+          java.util.concurrent.ExecutionException,
           java.util.concurrent.TimeoutException {
     // keeps track of whether we've loaded the file
     boolean loaded = false;
@@ -302,6 +305,7 @@ public void testUserAgentSuffixForInsertFileAPI() throws Exception {
    * <p>Creates the thread only two times since the manager was closed only once.
    */
   @Test
+  @Ignore // SNOW-957347: re-enable after fix
   public void testMultipleSimpleIngestManagers() throws Exception {
     // put
     TestUtils.executeQuery("put file://" + testFilePath + " @" + stageName);

From a06d406e0a355182d834521c8923647244e3b299 Mon Sep 17 00:00:00 2001
From: Dhara Shah <dhara.shah@snowflake.com>
Date: Mon, 13 Nov 2023 12:29:27 -0800
Subject: [PATCH 307/356] changed snapshot version from 2.0.4 to 2.0.5-SNAPSHOT
 (#650)

* changed snapshot version from 2.0.4 to 2.0.4-SNAPSHOT

* updated version from 2.0.4 to 2.0.5
---
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 90f5ccbb5..fa478ee08 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.0.4</version>
+  <version>2.0.5-SNAPSHOT</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 77443e7b8..605655c88 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.0.4";
+  public static final String DEFAULT_VERSION = "2.0.5-SNAPSHOT";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From e53d5dab321e3c0b4cdcbab3a00a3bd090fd79a9 Mon Sep 17 00:00:00 2001
From: Lorenz Thiede <lorenz.thiede@snowflake.com>
Date: Tue, 12 Dec 2023 11:22:57 +0100
Subject: [PATCH 308/356] SNOW-964536 benchmark setup (#655)

SNOW-964536 Add pipeline Jenkinsfile triggering jobs to setup TPC-DS BDEC datasets
---
 Jenkinsfile | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 Jenkinsfile

diff --git a/Jenkinsfile b/Jenkinsfile
new file mode 100644
index 000000000..f1a340a80
--- /dev/null
+++ b/Jenkinsfile
@@ -0,0 +1,42 @@
+pipeline {
+    agent { label 'parallelizable-c7' }
+    options { timestamps() }
+    environment {
+        ingest_sdk_dir = "${WORKSPACE}/snowflake-ingest-java"
+        ingest_sdk_tag = sh(returnStdout: true, script: "cd $ingest_sdk_dir && git describe --tags").trim()
+
+    }
+    stages {
+        stage('TriggerJobs') {
+            steps {
+                script {
+                    def valid_db_name_tag = ingest_sdk_tag.split('\\.').join('_')
+                    def deployments = [
+                            "qa3": {
+                                build job: "SFPerf-Other-Jobs/TPCDS_BDEC_Setup",
+                                        parameters: [
+                                                string(name: 'ingest_sdk_github_branch', value: ingest_sdk_tag),
+                                                string(name: 'database', value: "STREAMING_INGEST_BENCHMARK_DB_${valid_db_name_tag}"),
+                                                string(name: 'deployment', value: 'qa3.us-west-2.aws'),
+                                                string(name: 'tpcds_scale_factor', value: 'sf1000')
+                                        ],
+                                        propagate: true
+                            },
+                            "preprod12": {
+                                build job: "SFPerf-Other-Jobs/TPCDS_BDEC_Setup",
+                                        parameters: [
+                                                string(name: 'ingest_sdk_github_branch', value: ingest_sdk_tag),
+                                                string(name: 'database', value: "STREAMING_INGEST_BENCHMARK_DB_${valid_db_name_tag}"),
+                                                string(name: 'deployment', value: 'preprod12.us-west-2.aws'),
+                                                string(name: 'tpcds_scale_factor', value: 'sf1000')
+                                        ],
+                                        propagate: true
+                            }
+                    ]
+
+                    parallel deployments
+                }
+            }
+        }
+    }
+}
\ No newline at end of file

From 6be2e194c89ab14d498a02e43ed17fd4349e3a2f Mon Sep 17 00:00:00 2001
From: David Wang <63617111+sfc-gh-dwang@users.noreply.github.com>
Date: Fri, 15 Dec 2023 17:03:18 -0800
Subject: [PATCH 309/356] SNOW-984877 Update to support customized url and add
 customer name in request header (#656)

* SNOW-984877 Update to support customzed url parser and RequestBuilder

* update to set account name header by argument

* update format and compiling issue

* address comment
---
 .../ingest/connection/RequestBuilder.java     | 81 +++++++++++++++++++
 ...SnowflakeStreamingIngestClientFactory.java | 28 ++++++-
 ...nowflakeStreamingIngestClientInternal.java | 55 ++++++++++++-
 3 files changed, 159 insertions(+), 5 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 605655c88..9456ca2db 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -54,6 +54,10 @@ public class RequestBuilder {
   // whatever the actual host is
   private final String host;
 
+  private final String accountName;
+
+  private final boolean addAccountNameInRequest;
+
   private final String userAgentSuffix;
 
   // Reference to the telemetry service
@@ -123,6 +127,8 @@ public class RequestBuilder {
 
   public static final String HTTP_HEADER_CONTENT_TYPE_JSON = "application/json";
 
+  private static final String SF_HEADER_ACCOUNT_NAME = "Snowflake-Account";
+
   /**
    * RequestBuilder - general usage constructor
    *
@@ -206,6 +212,36 @@ public RequestBuilder(
         null);
   }
 
+  /**
+   * RequestBuilder - constructor used by streaming ingest
+   *
+   * @param url - the Snowflake account to which we're connecting
+   * @param userName - the username of the entity loading files
+   * @param credential - the credential we'll use to authenticate
+   * @param httpClient - reference to the http client
+   * @param clientName - name of the client, used to uniquely identify a client if used
+   */
+  public RequestBuilder(
+      SnowflakeURL url,
+      String userName,
+      Object credential,
+      CloseableHttpClient httpClient,
+      String clientName,
+      boolean addAccountNameInRequest) {
+    this(
+        url.getAccount(),
+        userName,
+        credential,
+        url.getScheme(),
+        url.getUrlWithoutPort(),
+        url.getPort(),
+        null,
+        null,
+        httpClient,
+        clientName,
+        addAccountNameInRequest);
+  }
+
   /**
    * RequestBuilder - constructor used by streaming ingest
    *
@@ -259,6 +295,46 @@ public RequestBuilder(
       SecurityManager securityManager,
       CloseableHttpClient httpClient,
       String clientName) {
+    this(accountName,
+        userName,
+        credential,
+        schemeName,
+        hostName,
+        portNum,
+        userAgentSuffix,
+        securityManager,
+        httpClient,
+        clientName,
+        false);
+  }
+
+  /**
+   * RequestBuilder - this constructor is for testing purposes only
+   *
+   * @param accountName - the account name to which we're connecting
+   * @param userName - for whom are we connecting?
+   * @param credential - our auth credentials, either JWT key pair or OAuth credential
+   * @param schemeName - are we HTTP or HTTPS?
+   * @param hostName - the host for this snowflake instance
+   * @param portNum - the port number
+   * @param userAgentSuffix - The suffix part of HTTP Header User-Agent
+   * @param securityManager - The security manager for authentication
+   * @param httpClient - reference to the http client
+   * @param clientName - name of the client, used to uniquely identify a client if used
+   * @param addAccountNameInRequest if ture, add account name in request header
+   */
+  public RequestBuilder(
+      String accountName,
+      String userName,
+      Object credential,
+      String schemeName,
+      String hostName,
+      int portNum,
+      String userAgentSuffix,
+      SecurityManager securityManager,
+      CloseableHttpClient httpClient,
+      String clientName,
+      boolean addAccountNameInRequest) {
     // none of these arguments should be null
     if (accountName == null || userName == null || credential == null) {
       throw new IllegalArgumentException();
@@ -281,6 +357,8 @@ public RequestBuilder(
     this.scheme = schemeName;
     this.host = hostName;
     this.userAgentSuffix = userAgentSuffix;
+    this.accountName = accountName;
+    this.addAccountNameInRequest = addAccountNameInRequest;
 
     // create our security/token manager
     if (securityManager == null) {
@@ -570,6 +648,9 @@ private static void addUserAgent(HttpUriRequest request, String userAgentSuffix)
   public void addToken(HttpUriRequest request) {
     request.setHeader(HttpHeaders.AUTHORIZATION, BEARER_PARAMETER + securityManager.getToken());
     request.setHeader(SF_HEADER_AUTHORIZATION_TOKEN_TYPE, this.securityManager.getTokenType());
+    if(addAccountNameInRequest) {
+      request.setHeader(SF_HEADER_ACCOUNT_NAME, accountName);
+    }
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java
index 2af794cd2..9ba87cffb 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java
@@ -6,6 +6,7 @@
 
 import java.util.Map;
 import java.util.Properties;
+
 import net.snowflake.ingest.streaming.internal.SnowflakeStreamingIngestClientInternal;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.SnowflakeURL;
@@ -28,6 +29,12 @@ public static class Builder {
     // Allows client to override some default parameter values
     private Map<String, Object> parameterOverrides;
 
+    // preset SnowflakeURL instance
+    private SnowflakeURL snowflakeURL;
+
+    // flag to specify if we need to add account name in the request header
+    private boolean addAccountNameInRequest;
+
     private Builder(String name) {
       this.name = name;
     }
@@ -37,6 +44,16 @@ public Builder setProperties(Properties prop) {
       return this;
     }
 
+    public Builder setSnowflakeURL(SnowflakeURL snowflakeURL) {
+      this.snowflakeURL = snowflakeURL;
+      return this;
+    }
+
+    public Builder setAddAccountNameInRequest(boolean addAccountNameInRequest) {
+      this.addAccountNameInRequest = addAccountNameInRequest;
+      return this;
+    }
+
     public Builder setParameterOverrides(Map<String, Object> parameterOverrides) {
       this.parameterOverrides = parameterOverrides;
       return this;
@@ -47,10 +64,17 @@ public SnowflakeStreamingIngestClient build() {
       Utils.assertNotNull("connection properties", this.prop);
 
       Properties prop = Utils.createProperties(this.prop);
-      SnowflakeURL accountURL = new SnowflakeURL(prop.getProperty(Constants.ACCOUNT_URL));
+      SnowflakeURL accountURL = this.snowflakeURL;
+      if (accountURL == null) {
+        accountURL = new SnowflakeURL(prop.getProperty(Constants.ACCOUNT_URL));
+      }
 
+      if (addAccountNameInRequest) {
+        return new SnowflakeStreamingIngestClientInternal<>(
+          this.name, accountURL, prop, this.parameterOverrides, addAccountNameInRequest);
+      }
       return new SnowflakeStreamingIngestClientInternal<>(
-          this.name, accountURL, prop, this.parameterOverrides);
+        this.name, accountURL, prop, this.parameterOverrides);
     }
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index a52149331..d857efdf9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -158,6 +158,30 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
       boolean isTestMode,
       RequestBuilder requestBuilder,
       Map<String, Object> parameterOverrides) {
+    this(name, accountURL, prop, httpClient, isTestMode, requestBuilder, parameterOverrides, false);
+  }
+
+  /**
+   * Constructor
+   *
+   * @param name the name of the client
+   * @param accountURL Snowflake account url
+   * @param prop connection properties
+   * @param httpClient http client for sending request
+   * @param isTestMode whether we're under test mode
+   * @param requestBuilder http request builder
+   * @param parameterOverrides parameters we override in case we want to set different values
+   * @param addAccountNameInRequest if true, will add account name in request header
+   */
+  SnowflakeStreamingIngestClientInternal(
+      String name,
+      SnowflakeURL accountURL,
+      Properties prop,
+      CloseableHttpClient httpClient,
+      boolean isTestMode,
+      RequestBuilder requestBuilder,
+      Map<String, Object> parameterOverrides,
+      boolean addAccountNameInRequest) {
     this.parameterProvider = new ParameterProvider(parameterOverrides, prop);
 
     this.name = name;
@@ -190,13 +214,13 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
                 prop.getProperty(Constants.OAUTH_CLIENT_SECRET),
                 prop.getProperty(Constants.OAUTH_REFRESH_TOKEN));
       }
-      this.requestBuilder =
-          new RequestBuilder(
+      this.requestBuilder = new RequestBuilder(
               accountURL,
               prop.get(USER).toString(),
               credential,
               this.httpClient,
-              String.format("%s_%s", this.name, System.currentTimeMillis()));
+              String.format("%s_%s", this.name, System.currentTimeMillis()),
+              addAccountNameInRequest);
 
       logger.logInfo("Using {} for authorization", this.requestBuilder.getAuthType());
 
@@ -236,6 +260,31 @@ public SnowflakeStreamingIngestClientInternal(
     this(name, accountURL, prop, null, false, null, parameterOverrides);
   }
 
+  /**
+   * Default Constructor
+   *
+   * @param name the name of the client
+   * @param accountURL Snowflake account url
+   * @param prop connection properties
+   * @param parameterOverrides map of parameters to override for this client
+   * @param addAccountNameInRequest if true, add account name in request header
+   */
+  public SnowflakeStreamingIngestClientInternal(
+      String name,
+      SnowflakeURL accountURL,
+      Properties prop,
+      Map<String, Object> parameterOverrides,
+      boolean addAccountNameInRequest) {
+    this(name,
+        accountURL,
+        prop,
+        null,
+        false,
+        null,
+        parameterOverrides,
+        addAccountNameInRequest);
+  }
+
   /**
    * Constructor for TEST ONLY
    *

From 764990012e75ab33fc3f111fe6af2bec1820e2b7 Mon Sep 17 00:00:00 2001
From: Gloria Doci <gloria.doci@snowflake.com>
Date: Mon, 18 Dec 2023 16:42:05 +0100
Subject: [PATCH 310/356] SNOW-928922 send spansMixedTables flag in blob
 registration requests (#651)

---
 .../streaming/internal/BlobMetadata.java      | 23 +++++++++++++++----
 .../streaming/internal/FlushService.java      |  9 +++++++-
 ...nowflakeStreamingIngestClientInternal.java |  5 +++-
 .../internal/RegisterServiceTest.java         |  4 +++-
 4 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
index d2cf31a6c..fb366db9c 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/BlobMetadata.java
@@ -18,11 +18,18 @@ class BlobMetadata {
   private final Constants.BdecVersion bdecVersion;
   private final List<ChunkMetadata> chunks;
   private final BlobStats blobStats;
+  private final boolean spansMixedTables;
 
   // used for testing only
   @VisibleForTesting
   BlobMetadata(String path, String md5, List<ChunkMetadata> chunks, BlobStats blobStats) {
-    this(path, md5, ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT, chunks, blobStats);
+    this(
+        path,
+        md5,
+        ParameterProvider.BLOB_FORMAT_VERSION_DEFAULT,
+        chunks,
+        blobStats,
+        chunks == null ? false : chunks.size() > 1);
   }
 
   BlobMetadata(
@@ -30,12 +37,14 @@ class BlobMetadata {
       String md5,
       Constants.BdecVersion bdecVersion,
       List<ChunkMetadata> chunks,
-      BlobStats blobStats) {
+      BlobStats blobStats,
+      boolean spansMixedTables) {
     this.path = path;
     this.md5 = md5;
     this.bdecVersion = bdecVersion;
     this.chunks = chunks;
     this.blobStats = blobStats;
+    this.spansMixedTables = spansMixedTables;
   }
 
   @JsonIgnore
@@ -68,13 +77,19 @@ BlobStats getBlobStats() {
     return this.blobStats;
   }
 
+  @JsonProperty("spans_mixed_tables")
+  boolean getSpansMixedTables() {
+    return this.spansMixedTables;
+  }
+
   /** Create {@link BlobMetadata}. */
   static BlobMetadata createBlobMetadata(
       String path,
       String md5,
       Constants.BdecVersion bdecVersion,
       List<ChunkMetadata> chunks,
-      BlobStats blobStats) {
-    return new BlobMetadata(path, md5, bdecVersion, chunks, blobStats);
+      BlobStats blobStats,
+      boolean spansMixedTables) {
+    return new BlobMetadata(path, md5, bdecVersion, chunks, blobStats, spansMixedTables);
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index cb1ef7810..f47580feb 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -568,8 +568,15 @@ BlobMetadata upload(
         blob.length,
         System.currentTimeMillis() - startTime);
 
+    // at this point we know for sure if the BDEC file has data for more than one chunk, i.e.
+    // spans mixed tables or not
     return BlobMetadata.createBlobMetadata(
-        blobPath, BlobBuilder.computeMD5(blob), bdecVersion, metadata, blobStats);
+        blobPath,
+        BlobBuilder.computeMD5(blob),
+        bdecVersion,
+        metadata,
+        blobStats,
+        metadata == null ? false : metadata.size() > 1);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index d857efdf9..88510a8d2 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -719,7 +719,10 @@ List<BlobMetadata> getRetryBlobs(
                     blobMetadata.getMD5(),
                     blobMetadata.getVersion(),
                     relevantChunks,
-                    blobMetadata.getBlobStats()));
+                    blobMetadata.getBlobStats(),
+                    // Important to not change the spansMixedTables value in case of retries. The
+                    // correct value is the value that the already uploaded blob has.
+                    blobMetadata.getSpansMixedTables()));
           }
         });
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
index 000b948e9..37eb5f96e 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java
@@ -7,6 +7,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import net.snowflake.ingest.utils.Pair;
@@ -17,7 +18,7 @@
 public class RegisterServiceTest {
 
   @Test
-  public void testRegisterService() {
+  public void testRegisterService() throws ExecutionException, InterruptedException {
     RegisterService<StubChunkData> rs = new RegisterService<>(null, true);
 
     Pair<FlushService.BlobData<StubChunkData>, CompletableFuture<BlobMetadata>> blobFuture =
@@ -26,6 +27,7 @@ public void testRegisterService() {
             CompletableFuture.completedFuture(new BlobMetadata("path", "md5", null, null)));
     rs.addBlobs(Collections.singletonList(blobFuture));
     Assert.assertEquals(1, rs.getBlobsList().size());
+    Assert.assertEquals(false, blobFuture.getValue().get().getSpansMixedTables());
     List<FlushService.BlobData<StubChunkData>> errorBlobs = rs.registerBlobs(null);
     Assert.assertEquals(0, rs.getBlobsList().size());
     Assert.assertEquals(0, errorBlobs.size());

From 8413e4adc1a201b930813fe0d9bcbccba8d65b9f Mon Sep 17 00:00:00 2001
From: Lorenz Thiede <lorenz.thiede@snowflake.com>
Date: Fri, 22 Dec 2023 18:17:23 +0100
Subject: [PATCH 311/356] SNOW-964536 Fix name of database the dataset is
 created in (#658)

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f1a340a80..eedd90a7b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -16,7 +16,7 @@ pipeline {
                                 build job: "SFPerf-Other-Jobs/TPCDS_BDEC_Setup",
                                         parameters: [
                                                 string(name: 'ingest_sdk_github_branch', value: ingest_sdk_tag),
-                                                string(name: 'database', value: "STREAMING_INGEST_BENCHMARK_DB_${valid_db_name_tag}"),
+                                                string(name: 'database', value: "BENCHMARK_DB_BDEC_PERFORMANCE_SIGNOFF_${valid_db_name_tag}"),
                                                 string(name: 'deployment', value: 'qa3.us-west-2.aws'),
                                                 string(name: 'tpcds_scale_factor', value: 'sf1000')
                                         ],
@@ -26,7 +26,7 @@ pipeline {
                                 build job: "SFPerf-Other-Jobs/TPCDS_BDEC_Setup",
                                         parameters: [
                                                 string(name: 'ingest_sdk_github_branch', value: ingest_sdk_tag),
-                                                string(name: 'database', value: "STREAMING_INGEST_BENCHMARK_DB_${valid_db_name_tag}"),
+                                                string(name: 'database', value: "BENCHMARK_DB_BDEC_PERFORMANCE_SIGNOFF_${valid_db_name_tag}"),
                                                 string(name: 'deployment', value: 'preprod12.us-west-2.aws'),
                                                 string(name: 'tpcds_scale_factor', value: 'sf1000')
                                         ],

From 04eea42d3e86a8f92dadc7ec69ac851b05d0c6f8 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Tue, 9 Jan 2024 00:58:47 -0800
Subject: [PATCH 312/356] NO-SNOW: Deprecate BUFFER_FLUSH_INTERVAL_IN_MILLIS
 parameter (#659)

- After MAX_CLIENT_LAG was introduced, it's not backward compatible with the current logic since MAX_CLIENT_LAG_ENABLED is true by default and we won't even check for BUFFER_FLUSH_INTERVAL_IN_MILLIS. Instead of supporting backward compatible, I decide to remove BUFFER_FLUSH_INTERVAL_IN_MILLIS completely since that is an undocumented parameter. We only support MAX_CLIENT_LAG going forward and using BUFFER_FLUSH_INTERVAL_IN_MILLIS will result in an exception.

- To match the behavior of BUFFER_FLUSH_INTERVAL_IN_MILLIS so the transition is smooth, MAX_CLIENT_LAG now also supports LONG input and input without a time unit (default will be milliseconds)

- Some code format changes done automatically by the Google Java Format

- Reenable some tests disabled long time ago
---
 .../ingest/connection/RequestBuilder.java     |   5 +-
 .../ingest/streaming/OpenChannelRequest.java  |   4 +-
 ...SnowflakeStreamingIngestClientFactory.java |   5 +-
 .../streaming/internal/FlushService.java      |   2 +-
 ...nowflakeStreamingIngestClientInternal.java |  14 +-
 .../ingest/utils/ParameterProvider.java       | 120 ++++++++----------
 .../net/snowflake/ingest/SimpleIngestIT.java  |   3 +-
 .../internal/ParameterProviderTest.java       |  89 ++++++-------
 .../SnowflakeStreamingIngestChannelTest.java  |  17 ++-
 .../SnowflakeStreamingIngestClientTest.java   |  14 +-
 .../streaming/internal/StreamingIngestIT.java |  25 ++--
 11 files changed, 137 insertions(+), 161 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 9456ca2db..869f734eb 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -295,7 +295,8 @@ public RequestBuilder(
       SecurityManager securityManager,
       CloseableHttpClient httpClient,
       String clientName) {
-    this(accountName,
+    this(
+        accountName,
         userName,
         credential,
         schemeName,
@@ -648,7 +649,7 @@ private static void addUserAgent(HttpUriRequest request, String userAgentSuffix)
   public void addToken(HttpUriRequest request) {
     request.setHeader(HttpHeaders.AUTHORIZATION, BEARER_PARAMETER + securityManager.getToken());
     request.setHeader(SF_HEADER_AUTHORIZATION_TOKEN_TYPE, this.securityManager.getTokenType());
-    if(addAccountNameInRequest) {
+    if (addAccountNameInRequest) {
       request.setHeader(SF_HEADER_ACCOUNT_NAME, accountName);
     }
   }
diff --git a/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java b/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java
index d30472b63..0cf16af2e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java
+++ b/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java
@@ -89,10 +89,10 @@ public OpenChannelRequestBuilder setDefaultTimezone(ZoneId defaultTimezone) {
       return this;
     }
 
-    public OpenChannelRequestBuilder setOffsetToken(String offsetToken){
+    public OpenChannelRequestBuilder setOffsetToken(String offsetToken) {
       this.offsetToken = offsetToken;
       this.isOffsetTokenProvided = true;
-      return  this;
+      return this;
     }
 
     public OpenChannelRequest build() {
diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java
index 9ba87cffb..ecab62432 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java
@@ -6,7 +6,6 @@
 
 import java.util.Map;
 import java.util.Properties;
-
 import net.snowflake.ingest.streaming.internal.SnowflakeStreamingIngestClientInternal;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.SnowflakeURL;
@@ -71,10 +70,10 @@ public SnowflakeStreamingIngestClient build() {
 
       if (addAccountNameInRequest) {
         return new SnowflakeStreamingIngestClientInternal<>(
-          this.name, accountURL, prop, this.parameterOverrides, addAccountNameInRequest);
+            this.name, accountURL, prop, this.parameterOverrides, addAccountNameInRequest);
       }
       return new SnowflakeStreamingIngestClientInternal<>(
-        this.name, accountURL, prop, this.parameterOverrides);
+          this.name, accountURL, prop, this.parameterOverrides);
     }
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index f47580feb..0e6998bdc 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -261,7 +261,7 @@ CompletableFuture<Void> flush(boolean isForce) {
             && !isTestMode()
             && (this.isNeedFlush
                 || timeDiffMillis
-                    >= this.owningClient.getParameterProvider().getBufferFlushIntervalInMs()))) {
+                    >= this.owningClient.getParameterProvider().getCachedMaxClientLagInMs()))) {
 
       return this.statsFuture()
           .thenCompose((v) -> this.distributeFlush(isForce, timeDiffMillis))
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 88510a8d2..47aa76d3b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -214,7 +214,8 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
                 prop.getProperty(Constants.OAUTH_CLIENT_SECRET),
                 prop.getProperty(Constants.OAUTH_REFRESH_TOKEN));
       }
-      this.requestBuilder = new RequestBuilder(
+      this.requestBuilder =
+          new RequestBuilder(
               accountURL,
               prop.get(USER).toString(),
               credential,
@@ -275,14 +276,7 @@ public SnowflakeStreamingIngestClientInternal(
       Properties prop,
       Map<String, Object> parameterOverrides,
       boolean addAccountNameInRequest) {
-    this(name,
-        accountURL,
-        prop,
-        null,
-        false,
-        null,
-        parameterOverrides,
-        addAccountNameInRequest);
+    this(name, accountURL, prop, null, false, null, parameterOverrides, addAccountNameInRequest);
   }
 
   /**
@@ -353,7 +347,7 @@ public SnowflakeStreamingIngestChannelInternal<?> openChannel(OpenChannelRequest
       payload.put("schema", request.getSchemaName());
       payload.put("write_mode", Constants.WriteMode.CLOUD_STORAGE.name());
       payload.put("role", this.role);
-      if (request.isOffsetTokenProvided()){
+      if (request.isOffsetTokenProvided()) {
         payload.put("offset_token", request.getOffsetToken());
       }
 
diff --git a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
index 82dd1164b..7967f1fdd 100644
--- a/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
+++ b/src/main/java/net/snowflake/ingest/utils/ParameterProvider.java
@@ -36,13 +36,10 @@ public class ParameterProvider {
 
   public static final String MAX_CLIENT_LAG = "MAX_CLIENT_LAG".toLowerCase();
 
-  public static final String MAX_CLIENT_LAG_ENABLED = "MAX_CLIENT_LAG_ENABLED".toLowerCase();
-
   public static final String BDEC_PARQUET_COMPRESSION_ALGORITHM =
       "BDEC_PARQUET_COMPRESSION_ALGORITHM".toLowerCase();
 
   // Default values
-  public static final long BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT = 1000;
   public static final long BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_DEFAULT = 100;
   public static final long INSERT_THROTTLE_INTERVAL_IN_MILLIS_DEFAULT = 1000;
   public static final int INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE_DEFAULT = 10;
@@ -57,12 +54,10 @@ public class ParameterProvider {
   public static final long MAX_CHUNK_SIZE_IN_BYTES_DEFAULT = 128000000L;
 
   // Lag related parameters
-  public static final String MAX_CLIENT_LAG_DEFAULT = "1 second";
-  public static final boolean MAX_CLIENT_LAG_ENABLED_DEFAULT = true;
-
+  public static final long MAX_CLIENT_LAG_DEFAULT = 1000; // 1 second
   static final long MAX_CLIENT_LAG_MS_MIN = TimeUnit.SECONDS.toMillis(1);
-
   static final long MAX_CLIENT_LAG_MS_MAX = TimeUnit.MINUTES.toMillis(10);
+
   public static final long MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT = 64 * 1024 * 1024; // 64 MB
   public static final int MAX_CHUNKS_IN_BLOB_AND_REGISTRATION_REQUEST_DEFAULT = 100;
 
@@ -114,11 +109,15 @@ private void updateValue(
    * @param props Properties file provided to client constructor
    */
   private void setParameterMap(Map<String, Object> parameterOverrides, Properties props) {
-    this.updateValue(
-        BUFFER_FLUSH_INTERVAL_IN_MILLIS,
-        BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT,
-        parameterOverrides,
-        props);
+    // BUFFER_FLUSH_INTERVAL_IN_MILLIS is deprecated and disallowed
+    if ((parameterOverrides != null
+            && parameterOverrides.containsKey(BUFFER_FLUSH_INTERVAL_IN_MILLIS))
+        || (props != null && props.containsKey(BUFFER_FLUSH_INTERVAL_IN_MILLIS))) {
+      throw new IllegalArgumentException(
+          String.format(
+              "%s is deprecated, please use %s instead",
+              BUFFER_FLUSH_INTERVAL_IN_MILLIS, MAX_CLIENT_LAG));
+    }
 
     this.updateValue(
         BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS,
@@ -177,8 +176,7 @@ private void setParameterMap(Map<String, Object> parameterOverrides, Properties
         MAX_CHUNK_SIZE_IN_BYTES, MAX_CHUNK_SIZE_IN_BYTES_DEFAULT, parameterOverrides, props);
 
     this.updateValue(MAX_CLIENT_LAG, MAX_CLIENT_LAG_DEFAULT, parameterOverrides, props);
-    this.updateValue(
-        MAX_CLIENT_LAG_ENABLED, MAX_CLIENT_LAG_ENABLED_DEFAULT, parameterOverrides, props);
+
     this.updateValue(
         MAX_CHUNKS_IN_BLOB_AND_REGISTRATION_REQUEST,
         MAX_CHUNKS_IN_BLOB_AND_REGISTRATION_REQUEST_DEFAULT,
@@ -193,57 +191,52 @@ private void setParameterMap(Map<String, Object> parameterOverrides, Properties
   }
 
   /** @return Longest interval in milliseconds between buffer flushes */
-  public long getBufferFlushIntervalInMs() {
-    if (getMaxClientLagEnabled()) {
-      if (cachedBufferFlushIntervalMs != -1L) {
-        return cachedBufferFlushIntervalMs;
-      }
-      long lag = getMaxClientLagMs();
-      if (cachedBufferFlushIntervalMs == -1L) {
-        cachedBufferFlushIntervalMs = lag;
-      }
+  public long getCachedMaxClientLagInMs() {
+    if (cachedBufferFlushIntervalMs != -1L) {
       return cachedBufferFlushIntervalMs;
-    } else {
-      Object val =
-          this.parameterMap.getOrDefault(
-              BUFFER_FLUSH_INTERVAL_IN_MILLIS, BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT);
-      return (val instanceof String) ? Long.parseLong(val.toString()) : (long) val;
     }
+
+    cachedBufferFlushIntervalMs = getMaxClientLagInMs();
+    return cachedBufferFlushIntervalMs;
   }
 
-  private long getMaxClientLagMs() {
+  private long getMaxClientLagInMs() {
     Object val = this.parameterMap.getOrDefault(MAX_CLIENT_LAG, MAX_CLIENT_LAG_DEFAULT);
-    if (!(val instanceof String)) {
-      return BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT;
-    }
-    String maxLag = (String) val;
-    String[] lagParts = maxLag.split(" ");
-    if (lagParts.length != 2
-        || (lagParts[0] == null || "".equals(lagParts[0]))
-        || (lagParts[1] == null || "".equals(lagParts[1]))) {
-      throw new IllegalArgumentException(
-          String.format("Failed to parse MAX_CLIENT_LAG = '%s'", maxLag));
-    }
-    long lag;
-    try {
-      lag = Long.parseLong(lagParts[0]);
-    } catch (Throwable t) {
-      throw new IllegalArgumentException(
-          String.format("Failed to parse MAX_CLIENT_LAG = '%s'", lagParts[0]), t);
-    }
     long computedLag;
-    switch (lagParts[1].toLowerCase()) {
-      case "second":
-      case "seconds":
-        computedLag = lag * TimeUnit.SECONDS.toMillis(1);
-        break;
-      case "minute":
-      case "minutes":
-        computedLag = lag * TimeUnit.SECONDS.toMillis(60);
-        break;
-      default:
+    if (val instanceof String) {
+      String maxLag = (String) val;
+      String[] lagParts = maxLag.split(" ");
+      if (lagParts.length > 2) {
         throw new IllegalArgumentException(
-            String.format("Invalid time unit supplied = '%s", lagParts[1]));
+            String.format("Failed to parse MAX_CLIENT_LAG = '%s'", maxLag));
+      }
+
+      // Compute the actual value
+      try {
+        computedLag = Long.parseLong(lagParts[0]);
+      } catch (Throwable t) {
+        throw new IllegalArgumentException(
+            String.format("Failed to parse MAX_CLIENT_LAG = '%s'", lagParts[0]), t);
+      }
+
+      // Compute the time unit if needed
+      if (lagParts.length == 2) {
+        switch (lagParts[1].toLowerCase()) {
+          case "second":
+          case "seconds":
+            computedLag = computedLag * TimeUnit.SECONDS.toMillis(1);
+            break;
+          case "minute":
+          case "minutes":
+            computedLag = computedLag * TimeUnit.SECONDS.toMillis(60);
+            break;
+          default:
+            throw new IllegalArgumentException(
+                String.format("Invalid time unit supplied = '%s", lagParts[1]));
+        }
+      }
+    } else {
+      computedLag = (long) val;
     }
 
     if (!(computedLag >= MAX_CLIENT_LAG_MS_MIN && computedLag <= MAX_CLIENT_LAG_MS_MAX)) {
@@ -253,13 +246,8 @@ private long getMaxClientLagMs() {
                   + " (milliseconds) = %s",
               MAX_CLIENT_LAG_MS_MIN, MAX_CLIENT_LAG_MS_MAX));
     }
-    return computedLag;
-  }
 
-  private boolean getMaxClientLagEnabled() {
-    Object val =
-        this.parameterMap.getOrDefault(MAX_CLIENT_LAG_ENABLED, MAX_CLIENT_LAG_ENABLED_DEFAULT);
-    return (val instanceof String) ? Boolean.parseBoolean(val.toString()) : (boolean) val;
+    return computedLag;
   }
 
   /** @return Time in milliseconds between checks to see if the buffer should be flushed */
@@ -411,8 +399,8 @@ public int getMaxChunksInBlobAndRegistrationRequest() {
   /** @return BDEC compression algorithm */
   public Constants.BdecParquetCompression getBdecParquetCompressionAlgorithm() {
     Object val =
-            this.parameterMap.getOrDefault(
-                    BDEC_PARQUET_COMPRESSION_ALGORITHM, BDEC_PARQUET_COMPRESSION_ALGORITHM_DEFAULT);
+        this.parameterMap.getOrDefault(
+            BDEC_PARQUET_COMPRESSION_ALGORITHM, BDEC_PARQUET_COMPRESSION_ALGORITHM_DEFAULT);
     if (val instanceof Constants.BdecParquetCompression) {
       return (Constants.BdecParquetCompression) val;
     }
diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
index ee73a2046..20a27eafb 100644
--- a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java
@@ -178,8 +178,7 @@ public void testSimpleIngestWithPattern() throws Exception {
   }
 
   private void getHistoryAndAssertLoad(SimpleIngestManager manager, String test_file_name_2)
-      throws InterruptedException,
-          java.util.concurrent.ExecutionException,
+      throws InterruptedException, java.util.concurrent.ExecutionException,
           java.util.concurrent.TimeoutException {
     // keeps track of whether we've loaded the file
     boolean loaded = false;
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
index 8499fc985..1ee6b9542 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
@@ -14,7 +14,7 @@ public class ParameterProviderTest {
 
   private Map<String, Object> getStartingParameterMap() {
     Map<String, Object> parameterMap = new HashMap<>();
-    parameterMap.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS, 3L);
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, 1000L);
     parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 4L);
     parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE, 6);
     parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_BYTES, 1024);
@@ -31,10 +31,9 @@ private Map<String, Object> getStartingParameterMap() {
   public void withValuesSet() {
     Properties prop = new Properties();
     Map<String, Object> parameterMap = getStartingParameterMap();
-    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, false);
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
 
-    Assert.assertEquals(3L, parameterProvider.getBufferFlushIntervalInMs());
+    Assert.assertEquals(1000L, parameterProvider.getCachedMaxClientLagInMs());
     Assert.assertEquals(4L, parameterProvider.getBufferFlushCheckIntervalInMs());
     Assert.assertEquals(6, parameterProvider.getInsertThrottleThresholdInPercentage());
     Assert.assertEquals(1024, parameterProvider.getInsertThrottleThresholdInBytes());
@@ -51,14 +50,13 @@ public void withValuesSet() {
   @Test
   public void withNullProps() {
     Map<String, Object> parameterMap = new HashMap<>();
-    parameterMap.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS, 3L);
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, 3000L);
     parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 4L);
     parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE, 6);
     parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_BYTES, 1024);
-    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, false);
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, null);
 
-    Assert.assertEquals(3, parameterProvider.getBufferFlushIntervalInMs());
+    Assert.assertEquals(3000, parameterProvider.getCachedMaxClientLagInMs());
     Assert.assertEquals(4, parameterProvider.getBufferFlushCheckIntervalInMs());
     Assert.assertEquals(6, parameterProvider.getInsertThrottleThresholdInPercentage());
     Assert.assertEquals(1024, parameterProvider.getInsertThrottleThresholdInBytes());
@@ -70,14 +68,13 @@ public void withNullProps() {
   @Test
   public void withNullParameterMap() {
     Properties props = new Properties();
-    props.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS, 3L);
+    props.put(ParameterProvider.MAX_CLIENT_LAG, 3000L);
     props.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 4L);
     props.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE, 6);
     props.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_BYTES, 1024);
-    props.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, false);
     ParameterProvider parameterProvider = new ParameterProvider(null, props);
 
-    Assert.assertEquals(3, parameterProvider.getBufferFlushIntervalInMs());
+    Assert.assertEquals(3000, parameterProvider.getCachedMaxClientLagInMs());
     Assert.assertEquals(4, parameterProvider.getBufferFlushCheckIntervalInMs());
     Assert.assertEquals(6, parameterProvider.getInsertThrottleThresholdInPercentage());
     Assert.assertEquals(1024, parameterProvider.getInsertThrottleThresholdInBytes());
@@ -91,8 +88,7 @@ public void withNullInputs() {
     ParameterProvider parameterProvider = new ParameterProvider(null, null);
 
     Assert.assertEquals(
-        ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT,
-        parameterProvider.getBufferFlushIntervalInMs());
+        ParameterProvider.MAX_CLIENT_LAG_DEFAULT, parameterProvider.getCachedMaxClientLagInMs());
     Assert.assertEquals(
         ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_DEFAULT,
         parameterProvider.getBufferFlushCheckIntervalInMs());
@@ -112,8 +108,7 @@ public void withDefaultValues() {
     ParameterProvider parameterProvider = new ParameterProvider();
 
     Assert.assertEquals(
-        ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS_DEFAULT,
-        parameterProvider.getBufferFlushIntervalInMs());
+        ParameterProvider.MAX_CLIENT_LAG_DEFAULT, parameterProvider.getCachedMaxClientLagInMs());
     Assert.assertEquals(
         ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS_DEFAULT,
         parameterProvider.getBufferFlushCheckIntervalInMs());
@@ -146,77 +141,75 @@ public void withDefaultValues() {
   public void testMaxClientLagEnabled() {
     Properties prop = new Properties();
     Map<String, Object> parameterMap = getStartingParameterMap();
-    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
     parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "2 second");
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
-    Assert.assertEquals(2000, parameterProvider.getBufferFlushIntervalInMs());
+    Assert.assertEquals(2000, parameterProvider.getCachedMaxClientLagInMs());
     // call again to trigger caching logic
-    Assert.assertEquals(2000, parameterProvider.getBufferFlushIntervalInMs());
+    Assert.assertEquals(2000, parameterProvider.getCachedMaxClientLagInMs());
   }
 
   @Test
   public void testMaxClientLagEnabledPluralTimeUnit() {
     Properties prop = new Properties();
     Map<String, Object> parameterMap = getStartingParameterMap();
-    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
     parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "2 seconds");
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
-    Assert.assertEquals(2000, parameterProvider.getBufferFlushIntervalInMs());
+    Assert.assertEquals(2000, parameterProvider.getCachedMaxClientLagInMs());
   }
 
   @Test
   public void testMaxClientLagEnabledMinuteTimeUnit() {
     Properties prop = new Properties();
     Map<String, Object> parameterMap = getStartingParameterMap();
-    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
     parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "1 minute");
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
-    Assert.assertEquals(60000, parameterProvider.getBufferFlushIntervalInMs());
+    Assert.assertEquals(60000, parameterProvider.getCachedMaxClientLagInMs());
   }
 
   @Test
   public void testMaxClientLagEnabledMinuteTimeUnitPluralTimeUnit() {
     Properties prop = new Properties();
     Map<String, Object> parameterMap = getStartingParameterMap();
-    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
     parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "2 minutes");
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
-    Assert.assertEquals(120000, parameterProvider.getBufferFlushIntervalInMs());
+    Assert.assertEquals(120000, parameterProvider.getCachedMaxClientLagInMs());
   }
 
   @Test
   public void testMaxClientLagEnabledDefaultValue() {
     Properties prop = new Properties();
     Map<String, Object> parameterMap = getStartingParameterMap();
-    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
-    Assert.assertEquals(1000, parameterProvider.getBufferFlushIntervalInMs());
+    Assert.assertEquals(
+        ParameterProvider.MAX_CLIENT_LAG_DEFAULT, parameterProvider.getCachedMaxClientLagInMs());
   }
 
   @Test
-  public void testMaxClientLagEnabledMissingUnit() {
+  public void testMaxClientLagEnabledDefaultUnit() {
     Properties prop = new Properties();
     Map<String, Object> parameterMap = getStartingParameterMap();
-    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
-    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "1");
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "3000");
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
-    try {
-      parameterProvider.getBufferFlushIntervalInMs();
-      Assert.fail("Should not have succeeded");
-    } catch (IllegalArgumentException e) {
-      Assert.assertTrue(e.getMessage().startsWith("Failed to parse"));
-    }
+    Assert.assertEquals(3000, parameterProvider.getCachedMaxClientLagInMs());
+  }
+
+  @Test
+  public void testMaxClientLagEnabledLongInput() {
+    Properties prop = new Properties();
+    Map<String, Object> parameterMap = getStartingParameterMap();
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, 3000L);
+    ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+    Assert.assertEquals(3000, parameterProvider.getCachedMaxClientLagInMs());
   }
 
   @Test
   public void testMaxClientLagEnabledMissingUnitTimeUnitSupplied() {
     Properties prop = new Properties();
     Map<String, Object> parameterMap = getStartingParameterMap();
-    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
     parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, " year");
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
     try {
-      parameterProvider.getBufferFlushIntervalInMs();
+      parameterProvider.getCachedMaxClientLagInMs();
       Assert.fail("Should not have succeeded");
     } catch (IllegalArgumentException e) {
       Assert.assertTrue(e.getMessage().startsWith("Failed to parse"));
@@ -227,11 +220,10 @@ public void testMaxClientLagEnabledMissingUnitTimeUnitSupplied() {
   public void testMaxClientLagEnabledInvalidTimeUnit() {
     Properties prop = new Properties();
     Map<String, Object> parameterMap = getStartingParameterMap();
-    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
     parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "1 year");
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
     try {
-      parameterProvider.getBufferFlushIntervalInMs();
+      parameterProvider.getCachedMaxClientLagInMs();
       Assert.fail("Should not have succeeded");
     } catch (IllegalArgumentException e) {
       Assert.assertTrue(e.getMessage().startsWith("Invalid time unit"));
@@ -242,11 +234,10 @@ public void testMaxClientLagEnabledInvalidTimeUnit() {
   public void testMaxClientLagEnabledInvalidUnit() {
     Properties prop = new Properties();
     Map<String, Object> parameterMap = getStartingParameterMap();
-    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
     parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "banana minute");
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
     try {
-      parameterProvider.getBufferFlushIntervalInMs();
+      parameterProvider.getCachedMaxClientLagInMs();
       Assert.fail("Should not have succeeded");
     } catch (IllegalArgumentException e) {
       Assert.assertTrue(e.getMessage().startsWith("Failed to parse"));
@@ -257,11 +248,10 @@ public void testMaxClientLagEnabledInvalidUnit() {
   public void testMaxClientLagEnabledThresholdBelow() {
     Properties prop = new Properties();
     Map<String, Object> parameterMap = getStartingParameterMap();
-    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
     parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "0 second");
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
     try {
-      parameterProvider.getBufferFlushIntervalInMs();
+      parameterProvider.getCachedMaxClientLagInMs();
       Assert.fail("Should not have succeeded");
     } catch (IllegalArgumentException e) {
       Assert.assertTrue(e.getMessage().startsWith("Lag falls outside"));
@@ -272,17 +262,30 @@ public void testMaxClientLagEnabledThresholdBelow() {
   public void testMaxClientLagEnabledThresholdAbove() {
     Properties prop = new Properties();
     Map<String, Object> parameterMap = getStartingParameterMap();
-    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG_ENABLED, true);
     parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "11 minutes");
     ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
     try {
-      parameterProvider.getBufferFlushIntervalInMs();
+      parameterProvider.getCachedMaxClientLagInMs();
       Assert.fail("Should not have succeeded");
     } catch (IllegalArgumentException e) {
       Assert.assertTrue(e.getMessage().startsWith("Lag falls outside"));
     }
   }
 
+  @Test
+  public void testMaxClientLagEnableEmptyInput() {
+    Properties prop = new Properties();
+    Map<String, Object> parameterMap = getStartingParameterMap();
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "");
+    ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+    try {
+      parameterProvider.getCachedMaxClientLagInMs();
+      Assert.fail("Should not have succeeded");
+    } catch (IllegalArgumentException e) {
+      Assert.assertEquals(e.getCause().getClass(), NumberFormatException.class);
+    }
+  }
+
   @Test
   public void testMaxChunksInBlobAndRegistrationRequest() {
     Properties prop = new Properties();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 037028086..528f9e46d 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -266,20 +266,19 @@ public void testOpenChannelRequestCreationSuccess() {
     Assert.assertFalse(request.isOffsetTokenProvided());
   }
 
-
   @Test
   public void testOpenChannelRequesCreationtWithOffsetToken() {
     OpenChannelRequest request =
-            OpenChannelRequest.builder("CHANNEL")
-                    .setDBName("STREAMINGINGEST_TEST")
-                    .setSchemaName("PUBLIC")
-                    .setTableName("T_STREAMINGINGEST")
-                    .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
-                    .setOffsetToken("TEST_TOKEN")
-                    .build();
+        OpenChannelRequest.builder("CHANNEL")
+            .setDBName("STREAMINGINGEST_TEST")
+            .setSchemaName("PUBLIC")
+            .setTableName("T_STREAMINGINGEST")
+            .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+            .setOffsetToken("TEST_TOKEN")
+            .build();
 
     Assert.assertEquals(
-            "STREAMINGINGEST_TEST.PUBLIC.T_STREAMINGINGEST", request.getFullyQualifiedTableName());
+        "STREAMINGINGEST_TEST.PUBLIC.T_STREAMINGINGEST", request.getFullyQualifiedTableName());
     Assert.assertEquals("TEST_TOKEN", request.getOffsetToken());
     Assert.assertTrue(request.isOffsetTokenProvided());
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index 11fc0b93b..cb75b3a52 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -142,17 +142,16 @@ public void setup() {
   }
 
   @Test
-  @Ignore // Until able to test in PROD
   public void testConstructorParameters() throws Exception {
     Properties prop = new Properties();
     prop.put(USER, TestUtils.getUser());
     prop.put(ACCOUNT_URL, TestUtils.getHost());
     prop.put(PRIVATE_KEY, TestUtils.getPrivateKey());
-    prop.put(ROLE, "role");
-    prop.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS, 123);
+    prop.put(ROLE, TestUtils.getRole());
+    prop.put(ParameterProvider.MAX_CLIENT_LAG, 1234);
 
     Map<String, Object> parameterMap = new HashMap<>();
-    parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 321);
+    parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 321L);
 
     SnowflakeStreamingIngestClientInternal<?> client =
         (SnowflakeStreamingIngestClientInternal<?>)
@@ -162,7 +161,7 @@ public void testConstructorParameters() throws Exception {
                 .build();
 
     Assert.assertEquals("client", client.getName());
-    Assert.assertEquals(123, client.getParameterProvider().getBufferFlushIntervalInMs());
+    Assert.assertEquals(1234, client.getParameterProvider().getCachedMaxClientLagInMs());
     Assert.assertEquals(321, client.getParameterProvider().getBufferFlushCheckIntervalInMs());
     Assert.assertEquals(
         ParameterProvider.INSERT_THROTTLE_INTERVAL_IN_MILLIS_DEFAULT,
@@ -256,15 +255,12 @@ public void testClientFactoryInvalidPrivateKey() throws Exception {
   }
 
   @Test
-  @Ignore // Wait for the client/configure endpoint to be available in PROD, can't mock the
-  // HttpUtil.executeGeneralRequest call because it's also used when setting up the
-  // connection
   public void testClientFactorySuccess() throws Exception {
     Properties prop = new Properties();
     prop.put(USER, TestUtils.getUser());
     prop.put(ACCOUNT_URL, TestUtils.getHost());
     prop.put(PRIVATE_KEY, TestUtils.getPrivateKey());
-    prop.put(ROLE, "role");
+    prop.put(ROLE, TestUtils.getRole());
 
     SnowflakeStreamingIngestClient client =
         SnowflakeStreamingIngestClientFactory.builder("client").setProperties(prop).build();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index eadbe81ee..3609b8e5e 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -192,7 +192,7 @@ public void testSimpleIngest() throws Exception {
   @Test
   public void testParameterOverrides() throws Exception {
     Map<String, Object> parameterMap = new HashMap<>();
-    parameterMap.put(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS, 30L);
+    parameterMap.put(ParameterProvider.MAX_CLIENT_LAG, "3 sec");
     parameterMap.put(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, 50L);
     parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE, 1);
     parameterMap.put(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_BYTES, 1024);
@@ -536,19 +536,16 @@ public void testMultiColumnIngest() throws Exception {
   public void testOpenChannelOffsetToken() throws Exception {
     String tableName = "offsetTokenTest";
     jdbcConnection
-            .createStatement()
-            .execute(
-                    String.format(
-                            "create or replace table %s (s text);",
-                            tableName));
+        .createStatement()
+        .execute(String.format("create or replace table %s (s text);", tableName));
     OpenChannelRequest request1 =
-            OpenChannelRequest.builder("TEST_CHANNEL")
-                    .setDBName(testDb)
-                    .setSchemaName(TEST_SCHEMA)
-                    .setTableName(tableName)
-                    .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
-                    .setOffsetToken("TEST_OFFSET")
-                    .build();
+        OpenChannelRequest.builder("TEST_CHANNEL")
+            .setDBName(testDb)
+            .setSchemaName(TEST_SCHEMA)
+            .setTableName(tableName)
+            .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+            .setOffsetToken("TEST_OFFSET")
+            .build();
 
     // Open a streaming ingest channel from the given client
     SnowflakeStreamingIngestChannel channel1 = client.openChannel(request1);
@@ -558,7 +555,7 @@ public void testOpenChannelOffsetToken() throws Exception {
 
     for (int i = 1; i < 15; i++) {
       if (channel1.getLatestCommittedOffsetToken() != null
-              && channel1.getLatestCommittedOffsetToken().equals("TEST_OFFSET")) {
+          && channel1.getLatestCommittedOffsetToken().equals("TEST_OFFSET")) {
         return;
       } else {
         Thread.sleep(2000);

From 11a18e7faeb76e4f1fadb0f89a816aff085faa2e Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Mon, 15 Jan 2024 15:44:03 +0100
Subject: [PATCH 313/356] NO-SNOW Run e2e jar tests on all clouds (#666)

---
 .github/workflows/End2EndTest.yml | 2 +-
 e2e-jar-test/pom.xml              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/End2EndTest.yml b/.github/workflows/End2EndTest.yml
index 91b16125e..8b492a6fc 100644
--- a/.github/workflows/End2EndTest.yml
+++ b/.github/workflows/End2EndTest.yml
@@ -74,7 +74,7 @@ jobs:
       fail-fast: false
       matrix:
         java: [ 8 ]
-        snowflake_cloud: [ 'AWS' ]
+        snowflake_cloud: [ 'AWS', 'AZURE', 'GCP' ]
     steps:
       - name: Checkout Code
         uses: actions/checkout@v2
diff --git a/e2e-jar-test/pom.xml b/e2e-jar-test/pom.xml
index 9a140aaeb..4c236b483 100644
--- a/e2e-jar-test/pom.xml
+++ b/e2e-jar-test/pom.xml
@@ -33,7 +33,7 @@
             <dependency>
                 <groupId>net.snowflake</groupId>
                 <artifactId>snowflake-jdbc-fips</artifactId>
-                <version>3.13.30</version>
+                <version>3.13.34</version>
             </dependency>
 
             <dependency>

From c2c4dd29dbcd274db15e503797b8dd49f25b08ea Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Wed, 17 Jan 2024 00:27:56 -0800
Subject: [PATCH 314/356] NO-SNOW: Revert one change that updates public API
 for internal use case (#662)

Revert 2b702db

This change updates the public API for an internal special use case only which is not ideal, looks like we keep updating public APIs in #661 as well. Given that we're still in a POC phase, I suggest we do everything in another branch instead, hence removing this from the next public release and could discuss what's the best way to support this internal use case once the POC is done.
---
 .../ingest/connection/RequestBuilder.java     | 82 -------------------
 ...SnowflakeStreamingIngestClientFactory.java | 31 ++-----
 ...nowflakeStreamingIngestClientInternal.java | 52 ++----------
 .../SnowflakeStreamingIngestClientTest.java   |  6 +-
 4 files changed, 18 insertions(+), 153 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 869f734eb..605655c88 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -54,10 +54,6 @@ public class RequestBuilder {
   // whatever the actual host is
   private final String host;
 
-  private final String accountName;
-
-  private final boolean addAccountNameInRequest;
-
   private final String userAgentSuffix;
 
   // Reference to the telemetry service
@@ -127,8 +123,6 @@ public class RequestBuilder {
 
   public static final String HTTP_HEADER_CONTENT_TYPE_JSON = "application/json";
 
-  private static final String SF_HEADER_ACCOUNT_NAME = "Snowflake-Account";
-
   /**
    * RequestBuilder - general usage constructor
    *
@@ -212,36 +206,6 @@ public RequestBuilder(
         null);
   }
 
-  /**
-   * RequestBuilder - constructor used by streaming ingest
-   *
-   * @param url - the Snowflake account to which we're connecting
-   * @param userName - the username of the entity loading files
-   * @param credential - the credential we'll use to authenticate
-   * @param httpClient - reference to the http client
-   * @param clientName - name of the client, used to uniquely identify a client if used
-   */
-  public RequestBuilder(
-      SnowflakeURL url,
-      String userName,
-      Object credential,
-      CloseableHttpClient httpClient,
-      String clientName,
-      boolean addAccountNameInRequest) {
-    this(
-        url.getAccount(),
-        userName,
-        credential,
-        url.getScheme(),
-        url.getUrlWithoutPort(),
-        url.getPort(),
-        null,
-        null,
-        httpClient,
-        clientName,
-        addAccountNameInRequest);
-  }
-
   /**
    * RequestBuilder - constructor used by streaming ingest
    *
@@ -295,47 +259,6 @@ public RequestBuilder(
       SecurityManager securityManager,
       CloseableHttpClient httpClient,
       String clientName) {
-    this(
-        accountName,
-        userName,
-        credential,
-        schemeName,
-        hostName,
-        portNum,
-        userAgentSuffix,
-        securityManager,
-        httpClient,
-        clientName,
-        false);
-  }
-
-  /**
-   * RequestBuilder - this constructor is for testing purposes only
-   *
-   * @param accountName - the account name to which we're connecting
-   * @param userName - for whom are we connecting?
-   * @param credential - our auth credentials, either JWT key pair or OAuth credential
-   * @param schemeName - are we HTTP or HTTPS?
-   * @param hostName - the host for this snowflake instance
-   * @param portNum - the port number
-   * @param userAgentSuffix - The suffix part of HTTP Header User-Agent
-   * @param securityManager - The security manager for authentication
-   * @param httpClient - reference to the http client
-   * @param clientName - name of the client, used to uniquely identify a client if used
-   * @param addAccountNameInRequest if ture, add account name in request header
-   */
-  public RequestBuilder(
-      String accountName,
-      String userName,
-      Object credential,
-      String schemeName,
-      String hostName,
-      int portNum,
-      String userAgentSuffix,
-      SecurityManager securityManager,
-      CloseableHttpClient httpClient,
-      String clientName,
-      boolean addAccountNameInRequest) {
     // none of these arguments should be null
     if (accountName == null || userName == null || credential == null) {
       throw new IllegalArgumentException();
@@ -358,8 +281,6 @@ public RequestBuilder(
     this.scheme = schemeName;
     this.host = hostName;
     this.userAgentSuffix = userAgentSuffix;
-    this.accountName = accountName;
-    this.addAccountNameInRequest = addAccountNameInRequest;
 
     // create our security/token manager
     if (securityManager == null) {
@@ -649,9 +570,6 @@ private static void addUserAgent(HttpUriRequest request, String userAgentSuffix)
   public void addToken(HttpUriRequest request) {
     request.setHeader(HttpHeaders.AUTHORIZATION, BEARER_PARAMETER + securityManager.getToken());
     request.setHeader(SF_HEADER_AUTHORIZATION_TOKEN_TYPE, this.securityManager.getTokenType());
-    if (addAccountNameInRequest) {
-      request.setHeader(SF_HEADER_ACCOUNT_NAME, accountName);
-    }
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java
index ecab62432..cd6d78787 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClientFactory.java
@@ -28,11 +28,8 @@ public static class Builder {
     // Allows client to override some default parameter values
     private Map<String, Object> parameterOverrides;
 
-    // preset SnowflakeURL instance
-    private SnowflakeURL snowflakeURL;
-
-    // flag to specify if we need to add account name in the request header
-    private boolean addAccountNameInRequest;
+    // Indicates whether it's under test mode
+    private boolean isTestMode;
 
     private Builder(String name) {
       this.name = name;
@@ -43,18 +40,13 @@ public Builder setProperties(Properties prop) {
       return this;
     }
 
-    public Builder setSnowflakeURL(SnowflakeURL snowflakeURL) {
-      this.snowflakeURL = snowflakeURL;
-      return this;
-    }
-
-    public Builder setAddAccountNameInRequest(boolean addAccountNameInRequest) {
-      this.addAccountNameInRequest = addAccountNameInRequest;
+    public Builder setParameterOverrides(Map<String, Object> parameterOverrides) {
+      this.parameterOverrides = parameterOverrides;
       return this;
     }
 
-    public Builder setParameterOverrides(Map<String, Object> parameterOverrides) {
-      this.parameterOverrides = parameterOverrides;
+    public Builder setIsTestMode(boolean isTestMode) {
+      this.isTestMode = isTestMode;
       return this;
     }
 
@@ -63,17 +55,10 @@ public SnowflakeStreamingIngestClient build() {
       Utils.assertNotNull("connection properties", this.prop);
 
       Properties prop = Utils.createProperties(this.prop);
-      SnowflakeURL accountURL = this.snowflakeURL;
-      if (accountURL == null) {
-        accountURL = new SnowflakeURL(prop.getProperty(Constants.ACCOUNT_URL));
-      }
+      SnowflakeURL accountURL = new SnowflakeURL(prop.getProperty(Constants.ACCOUNT_URL));
 
-      if (addAccountNameInRequest) {
-        return new SnowflakeStreamingIngestClientInternal<>(
-            this.name, accountURL, prop, this.parameterOverrides, addAccountNameInRequest);
-      }
       return new SnowflakeStreamingIngestClientInternal<>(
-          this.name, accountURL, prop, this.parameterOverrides);
+          this.name, accountURL, prop, this.parameterOverrides, this.isTestMode);
     }
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 47aa76d3b..071f5f7c4 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -158,30 +158,6 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
       boolean isTestMode,
       RequestBuilder requestBuilder,
       Map<String, Object> parameterOverrides) {
-    this(name, accountURL, prop, httpClient, isTestMode, requestBuilder, parameterOverrides, false);
-  }
-
-  /**
-   * Constructor
-   *
-   * @param name the name of the client
-   * @param accountURL Snowflake account url
-   * @param prop connection properties
-   * @param httpClient http client for sending request
-   * @param isTestMode whether we're under test mode
-   * @param requestBuilder http request builder
-   * @param parameterOverrides parameters we override in case we want to set different values
-   * @param addAccountNameInRequest if true, will add account name in request header
-   */
-  SnowflakeStreamingIngestClientInternal(
-      String name,
-      SnowflakeURL accountURL,
-      Properties prop,
-      CloseableHttpClient httpClient,
-      boolean isTestMode,
-      RequestBuilder requestBuilder,
-      Map<String, Object> parameterOverrides,
-      boolean addAccountNameInRequest) {
     this.parameterProvider = new ParameterProvider(parameterOverrides, prop);
 
     this.name = name;
@@ -220,8 +196,7 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
               prop.get(USER).toString(),
               credential,
               this.httpClient,
-              String.format("%s_%s", this.name, System.currentTimeMillis()),
-              addAccountNameInRequest);
+              String.format("%s_%s", this.name, System.currentTimeMillis()));
 
       logger.logInfo("Using {} for authorization", this.requestBuilder.getAuthType());
 
@@ -252,35 +227,18 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
    * @param accountURL Snowflake account url
    * @param prop connection properties
    * @param parameterOverrides map of parameters to override for this client
-   */
-  public SnowflakeStreamingIngestClientInternal(
-      String name,
-      SnowflakeURL accountURL,
-      Properties prop,
-      Map<String, Object> parameterOverrides) {
-    this(name, accountURL, prop, null, false, null, parameterOverrides);
-  }
-
-  /**
-   * Default Constructor
-   *
-   * @param name the name of the client
-   * @param accountURL Snowflake account url
-   * @param prop connection properties
-   * @param parameterOverrides map of parameters to override for this client
-   * @param addAccountNameInRequest if true, add account name in request header
+   * @param isTestMode indicates whether it's under test mode
    */
   public SnowflakeStreamingIngestClientInternal(
       String name,
       SnowflakeURL accountURL,
       Properties prop,
       Map<String, Object> parameterOverrides,
-      boolean addAccountNameInRequest) {
-    this(name, accountURL, prop, null, false, null, parameterOverrides, addAccountNameInRequest);
+      boolean isTestMode) {
+    this(name, accountURL, prop, null, isTestMode, null, parameterOverrides);
   }
 
-  /**
-   * Constructor for TEST ONLY
+  /*** Constructor for TEST ONLY
    *
    * @param name the name of the client
    */
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index cb75b3a52..0eece5f02 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -158,6 +158,7 @@ public void testConstructorParameters() throws Exception {
             SnowflakeStreamingIngestClientFactory.builder("client")
                 .setProperties(prop)
                 .setParameterOverrides(parameterMap)
+                .setIsTestMode(true)
                 .build();
 
     Assert.assertEquals("client", client.getName());
@@ -263,7 +264,10 @@ public void testClientFactorySuccess() throws Exception {
     prop.put(ROLE, TestUtils.getRole());
 
     SnowflakeStreamingIngestClient client =
-        SnowflakeStreamingIngestClientFactory.builder("client").setProperties(prop).build();
+        SnowflakeStreamingIngestClientFactory.builder("client")
+            .setProperties(prop)
+            .setIsTestMode(true)
+            .build();
 
     Assert.assertEquals("client", client.getName());
     Assert.assertFalse(client.isClosed());

From 0caaf7841a02583962066e4942ebc3975aea7cf1 Mon Sep 17 00:00:00 2001
From: Lukas Sembera <lukas.sembera@snowflake.com>
Date: Wed, 17 Jan 2024 22:25:55 +0100
Subject: [PATCH 315/356] SNOW-995369 GCS token refresh (#665)

---
 .../java/net/snowflake/IngestTestUtils.java   | 33 ++++++++++-
 .../java/net/snowflake/FipsIngestE2ETest.java | 13 +++-
 e2e-jar-test/pom.xml                          |  3 +-
 .../net/snowflake/StandardIngestE2ETest.java  | 14 ++++-
 .../streaming/internal/FlushService.java      |  6 +-
 .../internal/StreamingIngestStage.java        | 59 +++++++++++++++----
 .../internal/StreamingIngestStageTest.java    | 52 ++++++++++++----
 7 files changed, 147 insertions(+), 33 deletions(-)

diff --git a/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java b/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java
index 100972ea0..e7db8d1c2 100644
--- a/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java
+++ b/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java
@@ -13,6 +13,9 @@
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.Arrays;
 import java.util.Base64;
 import java.util.HashMap;
@@ -26,6 +29,8 @@
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class IngestTestUtils {
   private static final String PROFILE_PATH = "profile.json";
@@ -38,6 +43,8 @@ public class IngestTestUtils {
 
   private final String testId;
 
+  private static final Logger logger = LoggerFactory.getLogger(IngestTestUtils.class);
+
   private final SnowflakeStreamingIngestClient client;
 
   private final SnowflakeStreamingIngestChannel channel;
@@ -146,7 +153,7 @@ private void waitForOffset(SnowflakeStreamingIngestChannel channel, String expec
                     expectedOffset, lastCommittedOffset));
   }
 
-  public void test() throws InterruptedException {
+  public void runBasicTest() throws InterruptedException {
     // Insert few rows one by one
     for (int offset = 2; offset < 1000; offset++) {
       offset++;
@@ -161,6 +168,30 @@ public void test() throws InterruptedException {
     waitForOffset(channel, offset);
   }
 
+  public void runLongRunningTest(Duration testDuration) throws InterruptedException {
+    final Instant testStart = Instant.now();
+    int counter = 0;
+    while(true) {
+      counter++;
+
+      channel.insertRow(createRow(), String.valueOf(counter));
+
+      if (!channel.isValid()) {
+        throw new IllegalStateException("Channel has been invalidated");
+      }
+      Thread.sleep(60000);
+
+      final Duration elapsed = Duration.between(testStart, Instant.now());
+
+      logger.info("Test loop_nr={} duration={}s/{}s committed_offset={}", counter, elapsed.get(ChronoUnit.SECONDS), testDuration.get(ChronoUnit.SECONDS), channel.getLatestCommittedOffsetToken());
+
+      if (elapsed.compareTo(testDuration) > 0) {
+        break;
+      }
+    }
+    waitForOffset(channel, String.valueOf(counter));
+  }
+
   public void close() throws Exception {
     connection.close();
     channel.close().get();
diff --git a/e2e-jar-test/fips/src/test/java/net/snowflake/FipsIngestE2ETest.java b/e2e-jar-test/fips/src/test/java/net/snowflake/FipsIngestE2ETest.java
index 7279f23ff..c6f9bfe33 100644
--- a/e2e-jar-test/fips/src/test/java/net/snowflake/FipsIngestE2ETest.java
+++ b/e2e-jar-test/fips/src/test/java/net/snowflake/FipsIngestE2ETest.java
@@ -3,9 +3,12 @@
 import org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import java.security.Security;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
 
 public class FipsIngestE2ETest {
 
@@ -25,7 +28,13 @@ public void tearDown() throws Exception {
   }
 
   @Test
-  public void name() throws InterruptedException {
-    ingestTestUtils.test();
+  public void basicTest() throws InterruptedException {
+    ingestTestUtils.runBasicTest();
+  }
+
+  @Test
+  @Ignore("Takes too long to run")
+  public void longRunningTest() throws InterruptedException {
+    ingestTestUtils.runLongRunningTest(Duration.of(80, ChronoUnit.MINUTES));
   }
 }
diff --git a/e2e-jar-test/pom.xml b/e2e-jar-test/pom.xml
index 4c236b483..e4ba3635b 100644
--- a/e2e-jar-test/pom.xml
+++ b/e2e-jar-test/pom.xml
@@ -27,7 +27,8 @@
             <dependency>
                 <groupId>net.snowflake</groupId>
                 <artifactId>snowflake-ingest-sdk</artifactId>
-                <version>2.0.4</version>
+                <!-- This value should be the same as the version in the pom.xml of the SDK -->
+                <version>2.0.5-SNAPSHOT</version>
             </dependency>
 
             <dependency>
diff --git a/e2e-jar-test/standard/src/test/java/net/snowflake/StandardIngestE2ETest.java b/e2e-jar-test/standard/src/test/java/net/snowflake/StandardIngestE2ETest.java
index 255577655..211c421fc 100644
--- a/e2e-jar-test/standard/src/test/java/net/snowflake/StandardIngestE2ETest.java
+++ b/e2e-jar-test/standard/src/test/java/net/snowflake/StandardIngestE2ETest.java
@@ -2,8 +2,12 @@
 
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
+
 public class StandardIngestE2ETest {
 
   private IngestTestUtils ingestTestUtils;
@@ -19,7 +23,13 @@ public void tearDown() throws Exception {
   }
 
   @Test
-  public void name() throws InterruptedException {
-    ingestTestUtils.test();
+  public void basicTest() throws InterruptedException {
+    ingestTestUtils.runBasicTest();
+  }
+
+  @Test
+  @Ignore("Takes too long to run")
+  public void longRunningTest() throws InterruptedException {
+    ingestTestUtils.runLongRunningTest(Duration.of(80, ChronoUnit.MINUTES));
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 0e6998bdc..2324adda8 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -59,6 +59,9 @@
  */
 class FlushService<T> {
 
+  // The max number of upload retry attempts to the stage
+  private static final int DEFAULT_MAX_UPLOAD_RETRIES = 5;
+
   // Static class to save the list of channels that are used to build a blob, which is mainly used
   // to invalidate all the channels when there is a failure
   static class BlobData<T> {
@@ -163,7 +166,8 @@ List<List<ChannelData<T>>> getData() {
               client.getRole(),
               client.getHttpClient(),
               client.getRequestBuilder(),
-              client.getName());
+              client.getName(),
+              DEFAULT_MAX_UPLOAD_RETRIES);
     } catch (SnowflakeSQLException | IOException err) {
       throw new SFException(err, ErrorCode.UNABLE_TO_CONNECT_TO_STAGE);
     }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index 0d7e3f211..443097550 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -34,6 +34,7 @@
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
+import net.snowflake.client.jdbc.internal.google.cloud.storage.StorageException;
 import net.snowflake.ingest.connection.IngestResponseException;
 import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.utils.ErrorCode;
@@ -46,7 +47,6 @@ class StreamingIngestStage {
   private static final ObjectMapper mapper = new ObjectMapper();
   private static final long REFRESH_THRESHOLD_IN_MS =
       TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES);
-  static final int MAX_RETRY_COUNT = 1;
 
   private static final Logging logger = new Logging(StreamingIngestStage.class);
 
@@ -86,6 +86,8 @@ state to record unknown age.
   private final String clientName;
   private String clientPrefix;
 
+  private final int maxUploadRetries;
+
   // Proxy parameters that we set while calling the Snowflake JDBC to upload the streams
   private final Properties proxyProperties;
 
@@ -94,13 +96,15 @@ state to record unknown age.
       String role,
       CloseableHttpClient httpClient,
       RequestBuilder requestBuilder,
-      String clientName)
+      String clientName,
+      int maxUploadRetries)
       throws SnowflakeSQLException, IOException {
     this.httpClient = httpClient;
     this.role = role;
     this.requestBuilder = requestBuilder;
     this.clientName = clientName;
     this.proxyProperties = generateProxyPropertiesForJDBC();
+    this.maxUploadRetries = maxUploadRetries;
 
     if (!isTestMode) {
       refreshSnowflakeMetadata();
@@ -123,9 +127,10 @@ state to record unknown age.
       CloseableHttpClient httpClient,
       RequestBuilder requestBuilder,
       String clientName,
-      SnowflakeFileTransferMetadataWithAge testMetadata)
+      SnowflakeFileTransferMetadataWithAge testMetadata,
+      int maxRetryCount)
       throws SnowflakeSQLException, IOException {
-    this(isTestMode, role, httpClient, requestBuilder, clientName);
+    this(isTestMode, role, httpClient, requestBuilder, clientName, maxRetryCount);
     if (!isTestMode) {
       throw new SFException(ErrorCode.INTERNAL_ERROR);
     }
@@ -187,17 +192,46 @@ private void putRemote(String fullFilePath, byte[] data, int retryCount)
               .setProxyProperties(this.proxyProperties)
               .setDestFileName(fullFilePath)
               .build());
-    } catch (SnowflakeSQLException e) {
-      if (e.getErrorCode() != CLOUD_STORAGE_CREDENTIALS_EXPIRED || retryCount >= MAX_RETRY_COUNT) {
+    } catch (Exception e) {
+      if (retryCount >= maxUploadRetries) {
         logger.logError(
-            "Failed to upload to stage, client={}, message={}", clientName, e.getMessage());
-        throw e;
+            "Failed to upload to stage, retry attempts exhausted ({}), client={}, message={}",
+            maxUploadRetries,
+            clientName,
+            e.getMessage());
+        throw new SFException(e, ErrorCode.IO_ERROR);
       }
-      this.refreshSnowflakeMetadata();
-      this.putRemote(fullFilePath, data, ++retryCount);
-    } catch (Exception e) {
-      throw new SFException(e, ErrorCode.IO_ERROR);
+
+      if (isCredentialsExpiredException(e)) {
+        logger.logInfo(
+            "Stage metadata need to be refreshed due to upload error: {}", e.getMessage());
+        this.refreshSnowflakeMetadata();
+      }
+      retryCount++;
+      StreamingIngestUtils.sleepForRetry(retryCount);
+      logger.logInfo(
+          "Retrying upload, attempt {}/{} {}", retryCount, maxUploadRetries, e.getMessage());
+      this.putRemote(fullFilePath, data, retryCount);
+    }
+  }
+
+  /**
+   * @return Whether the passed exception means that credentials expired and the stage metadata
+   *     should be refreshed from Snowflake. The reasons for refresh is SnowflakeSQLException with
+   *     error code 240001 (thrown by the JDBC driver) or GCP StorageException with HTTP status 401.
+   */
+  static boolean isCredentialsExpiredException(Exception e) {
+    if (e == null || e.getClass() == null) {
+      return false;
     }
+
+    if (e instanceof SnowflakeSQLException) {
+      return ((SnowflakeSQLException) e).getErrorCode() == CLOUD_STORAGE_CREDENTIALS_EXPIRED;
+    } else if (e instanceof StorageException) {
+      return ((StorageException) e).getCode() == 401;
+    }
+
+    return false;
   }
 
   SnowflakeFileTransferMetadataWithAge refreshSnowflakeMetadata()
@@ -399,7 +433,6 @@ void putLocal(String fullFilePath, byte[] data) {
       String stageLocation = this.fileTransferMetadataWithAge.localLocation;
       File destFile = Paths.get(stageLocation, fullFilePath).toFile();
       FileUtils.copyInputStreamToFile(input, destFile);
-      System.out.println("Filename: " + destFile); // TODO @rcheng - remove this before merge
     } catch (Exception ex) {
       throw new SFException(ex, ErrorCode.BLOB_UPLOAD_FAILURE);
     }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
index a137ab9ed..a14f1c46b 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
@@ -1,6 +1,7 @@
 package net.snowflake.ingest.streaming.internal;
 
 import static net.snowflake.client.core.Constants.CLOUD_STORAGE_CREDENTIALS_EXPIRED;
+import static net.snowflake.ingest.streaming.internal.StreamingIngestStage.isCredentialsExpiredException;
 import static net.snowflake.ingest.utils.HttpUtil.HTTP_PROXY_PASSWORD;
 import static net.snowflake.ingest.utils.HttpUtil.HTTP_PROXY_USER;
 import static net.snowflake.ingest.utils.HttpUtil.NON_PROXY_HOSTS;
@@ -39,11 +40,13 @@
 import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
+import net.snowflake.client.jdbc.internal.google.cloud.storage.StorageException;
 import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
 import net.snowflake.ingest.TestUtils;
 import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ParameterProvider;
+import net.snowflake.ingest.utils.SFException;
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -121,7 +124,8 @@ public void testPutRemote() throws Exception {
             null,
             "clientName",
             new StreamingIngestStage.SnowflakeFileTransferMetadataWithAge(
-                originalMetadata, Optional.of(System.currentTimeMillis())));
+                originalMetadata, Optional.of(System.currentTimeMillis())),
+            1);
     PowerMockito.mockStatic(SnowflakeFileTransferAgent.class);
 
     final ArgumentCaptor<SnowflakeFileTransferConfig> captor =
@@ -163,7 +167,8 @@ public void testPutLocal() throws Exception {
                 null,
                 "clientName",
                 new StreamingIngestStage.SnowflakeFileTransferMetadataWithAge(
-                    fullFilePath, Optional.of(System.currentTimeMillis()))));
+                    fullFilePath, Optional.of(System.currentTimeMillis())),
+                1));
     Mockito.doReturn(true).when(stage).isLocalFS();
 
     stage.put(fileName, dataBytes);
@@ -174,7 +179,8 @@ public void testPutLocal() throws Exception {
   }
 
   @Test
-  public void testPutRemoteRefreshes() throws Exception {
+  public void doTestPutRemoteRefreshes() throws Exception {
+    int maxUploadRetryCount = 2;
     JsonNode exampleJson = mapper.readTree(exampleRemoteMeta);
     SnowflakeFileTransferMetadataV1 originalMetadata =
         (SnowflakeFileTransferMetadataV1)
@@ -190,7 +196,8 @@ public void testPutRemoteRefreshes() throws Exception {
             null,
             "clientName",
             new StreamingIngestStage.SnowflakeFileTransferMetadataWithAge(
-                originalMetadata, Optional.of(System.currentTimeMillis())));
+                originalMetadata, Optional.of(System.currentTimeMillis())),
+            maxUploadRetryCount);
     PowerMockito.mockStatic(SnowflakeFileTransferAgent.class);
     SnowflakeSQLException e =
         new SnowflakeSQLException(
@@ -202,16 +209,15 @@ public void testPutRemoteRefreshes() throws Exception {
 
     try {
       stage.putRemote("test/path", dataBytes);
-      Assert.assertTrue(false);
-    } catch (SnowflakeSQLException ex) {
+      Assert.fail("Should not succeed");
+    } catch (SFException ex) {
       // Expected behavior given mocked response
     }
-    PowerMockito.verifyStatic(
-        SnowflakeFileTransferAgent.class, times(StreamingIngestStage.MAX_RETRY_COUNT + 1));
+    PowerMockito.verifyStatic(SnowflakeFileTransferAgent.class, times(maxUploadRetryCount + 1));
     SnowflakeFileTransferAgent.uploadWithoutConnection(captor.capture());
     SnowflakeFileTransferConfig capturedConfig = captor.getValue();
 
-    Assert.assertEquals(false, capturedConfig.getRequireCompress());
+    Assert.assertFalse(capturedConfig.getRequireCompress());
     Assert.assertEquals(OCSPMode.FAIL_OPEN, capturedConfig.getOcspMode());
 
     SnowflakeFileTransferMetadataV1 capturedMetadata =
@@ -245,7 +251,8 @@ public void testPutRemoteGCS() throws Exception {
                 null,
                 "clientName",
                 new StreamingIngestStage.SnowflakeFileTransferMetadataWithAge(
-                    originalMetadata, Optional.of(System.currentTimeMillis()))));
+                    originalMetadata, Optional.of(System.currentTimeMillis())),
+                1));
     PowerMockito.mockStatic(SnowflakeFileTransferAgent.class);
     SnowflakeFileTransferMetadataV1 metaMock = Mockito.mock(SnowflakeFileTransferMetadataV1.class);
 
@@ -273,7 +280,7 @@ public void testRefreshSnowflakeMetadataRemote() throws Exception {
 
     ParameterProvider parameterProvider = new ParameterProvider();
     StreamingIngestStage stage =
-        new StreamingIngestStage(true, "role", mockClient, mockBuilder, "clientName");
+        new StreamingIngestStage(true, "role", mockClient, mockBuilder, "clientName", 1);
 
     StreamingIngestStage.SnowflakeFileTransferMetadataWithAge metadataWithAge =
         stage.refreshSnowflakeMetadata(true);
@@ -314,7 +321,7 @@ public void testFetchSignedURL() throws Exception {
     Mockito.when(mockClient.execute(Mockito.any())).thenReturn(mockResponse);
 
     StreamingIngestStage stage =
-        new StreamingIngestStage(true, "role", mockClient, mockBuilder, "clientName");
+        new StreamingIngestStage(true, "role", mockClient, mockBuilder, "clientName", 1);
 
     SnowflakeFileTransferMetadataV1 metadata = stage.fetchSignedURL("path/fileName");
 
@@ -359,7 +366,8 @@ public void testRefreshSnowflakeMetadataSynchronized() throws Exception {
             mockBuilder,
             "clientName",
             new StreamingIngestStage.SnowflakeFileTransferMetadataWithAge(
-                originalMetadata, Optional.of(0L)));
+                originalMetadata, Optional.of(0L)),
+            1);
 
     ThreadFactory buildUploadThreadFactory =
         new ThreadFactoryBuilder().setNameFormat("ingest-build-upload-thread-%d").build();
@@ -476,4 +484,22 @@ public void testShouldBypassProxy() {
       System.setProperty(NON_PROXY_HOSTS, oldNonProxyHosts);
     }
   }
+
+  @Test
+  public void testIsCredentialExpiredException() {
+    Assert.assertTrue(
+        isCredentialsExpiredException(
+            new SnowflakeSQLException("Error", CLOUD_STORAGE_CREDENTIALS_EXPIRED)));
+    Assert.assertTrue(isCredentialsExpiredException(new StorageException(401, "unauthorized")));
+
+    Assert.assertFalse(isCredentialsExpiredException(new StorageException(400, "bad request")));
+    Assert.assertFalse(isCredentialsExpiredException(null));
+    Assert.assertFalse(isCredentialsExpiredException(new RuntimeException()));
+    Assert.assertFalse(
+        isCredentialsExpiredException(
+            new RuntimeException(String.valueOf(CLOUD_STORAGE_CREDENTIALS_EXPIRED))));
+    Assert.assertFalse(
+        isCredentialsExpiredException(
+            new SnowflakeSQLException("Error", CLOUD_STORAGE_CREDENTIALS_EXPIRED + 1)));
+  }
 }

From bee1db1eda93dfcaf1ef61b84ff5af7127016dfa Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Thu, 18 Jan 2024 20:46:53 -0800
Subject: [PATCH 316/356] V2.0.5 release  (#664)

V2.0.5 release
---
 e2e-jar-test/pom.xml                          |  3 +--
 pom.xml                                       | 26 +++++++++----------
 .../ingest/connection/RequestBuilder.java     |  2 +-
 3 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/e2e-jar-test/pom.xml b/e2e-jar-test/pom.xml
index e4ba3635b..1012be2df 100644
--- a/e2e-jar-test/pom.xml
+++ b/e2e-jar-test/pom.xml
@@ -27,8 +27,7 @@
             <dependency>
                 <groupId>net.snowflake</groupId>
                 <artifactId>snowflake-ingest-sdk</artifactId>
-                <!-- This value should be the same as the version in the pom.xml of the SDK -->
-                <version>2.0.5-SNAPSHOT</version>
+                <version>2.0.5</version>
             </dependency>
 
             <dependency>
diff --git a/pom.xml b/pom.xml
index fa478ee08..23e6243f4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.0.5-SNAPSHOT</version>
+  <version>2.0.5</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
@@ -779,9 +779,9 @@
         <configuration>
           <errorRemedy>failFast</errorRemedy>
           <!--
-                    The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please
-                    check your dependencies", verify the conditions of the license and add the reference to it here.
-                    -->
+                               The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please
+                              check your dependencies", verify the conditions of the license and add the reference to it here.
+                            -->
           <includedLicenses>
             <includedLicense>Apache License 2.0</includedLicense>
             <includedLicense>BSD 2-Clause License</includedLicense>
@@ -899,9 +899,9 @@
       <build>
         <plugins>
           <!--
-                    Copy all project dependencies to target/dependency-jars. License processing Python script will look here for
-                    license files of SDK dependencies.
-                    -->
+                                        Copy all project dependencies to target/dependency-jars. License processing Python script will look here for
+                                        license files of SDK dependencies.
+                                        -->
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-dependency-plugin</artifactId>
@@ -923,9 +923,9 @@
           </plugin>
 
           <!--
-                    Compile the list of SDK dependencies in 'compile' and 'runtime' scopes.
-                    This list is an entry point for the license processing python script.
-                    -->
+                                        Compile the list of SDK dependencies in 'compile' and 'runtime' scopes.
+                                        This list is an entry point for the license processing python script.
+                                        -->
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-dependency-plugin</artifactId>
@@ -1108,9 +1108,9 @@
             </executions>
           </plugin>
           <!--
-                    Plugin executes license processing Python script, which copies third party license files into the directory
-                    target/generated-licenses-info/META-INF/third-party-licenses, which is then included in the shaded JAR.
-                    -->
+                                        Plugin executes license processing Python script, which copies third party license files into the directory
+                                        target/generated-licenses-info/META-INF/third-party-licenses, which is then included in the shaded JAR.
+                                        -->
           <plugin>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>exec-maven-plugin</artifactId>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 605655c88..26911501b 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.0.5-SNAPSHOT";
+  public static final String DEFAULT_VERSION = "2.0.5";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 0470633bfb2b0505c22eb7ffa8ba5cafc2cacf16 Mon Sep 17 00:00:00 2001
From: Xin Huang <xin.huang@snowflake.com>
Date: Mon, 22 Jan 2024 09:48:52 -0800
Subject: [PATCH 317/356] NO-SNOW Correct invalid use of javadoc (#668)

Correct invalid use of javadoc
---
 .../ingest/streaming/SnowflakeStreamingIngestChannel.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
index a8bb5db16..bb372c64c 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
@@ -260,7 +260,7 @@ InsertValidationResponse insertRows(
    * time of a channel open event. The schema may be changed on the Snowflake side in which case
    * this will continue to show an old schema version until the channel is re-opened.
    *
-   * @return map representing Column Name -> Column Properties
+   * @return map representing Column Name to Column Properties
    */
   Map<String, ColumnProperties> getTableSchema();
 }

From 213c81543c59ce8e09c45fad3a62904712e2a662 Mon Sep 17 00:00:00 2001
From: Lorenz Thiede <lorenz.thiede@snowflake.com>
Date: Thu, 1 Feb 2024 10:49:53 +0100
Subject: [PATCH 318/356] SNOW-983635 Allow ZSTD compression algorithm (#654)

---
 README.md                                        |  1 +
 pom.xml                                          | 13 +++++++------
 public_pom.xml                                   |  6 ++++++
 .../net/snowflake/ingest/utils/Constants.java    |  3 ++-
 .../internal/ParameterProviderTest.java          | 13 ++++++++++++-
 .../internal/StreamingIngestBigFilesIT.java      | 15 +++++++++++++++
 .../streaming/internal/StreamingIngestIT.java    | 16 +++++++++++++++-
 .../internal/datatypes/AbstractDataTypeTest.java | 15 +++++++++++++++
 8 files changed, 73 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 5027f3216..fcf935d51 100644
--- a/README.md
+++ b/README.md
@@ -17,6 +17,7 @@ The Snowflake Ingest Service SDK depends on the following libraries:
 
 * snowflake-jdbc (3.13.30 to 3.13.33) 
 * slf4j-api
+* com.github.luben:zstd-jni (1.5.0-1)
 
 These dependencies will be fetched automatically by build systems like Maven or Gradle. If you don't build your project
 using a build system, please make sure these dependencies are on the classpath.
diff --git a/pom.xml b/pom.xml
index 23e6243f4..47a1663d9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -356,7 +356,6 @@
 
   <!-- All of our needed dependencies -->
   <dependencies>
-
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-annotations</artifactId>
@@ -389,7 +388,6 @@
         </exclusion>
       </exclusions>
     </dependency>
-
     <dependency>
       <groupId>com.google.code.findbugs</groupId>
       <artifactId>jsr305</artifactId>
@@ -473,6 +471,12 @@
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.github.luben</groupId>
+      <artifactId>zstd-jni</artifactId>
+      <version>1.5.0-1</version>
+      <scope>runtime</scope>
+    </dependency>
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
@@ -954,6 +958,7 @@
                 <excludes>
                   <exclude>net.snowflake:snowflake-jdbc</exclude>
                   <exclude>org.slf4j:slf4j-api</exclude>
+                  <exclude>com.github.luben:zstd-jni</exclude>
                 </excludes>
               </artifactSet>
               <relocations>
@@ -1034,10 +1039,6 @@
                   <pattern>com.ctc</pattern>
                   <shadedPattern>${shadeBase}.com.ctc</shadedPattern>
                 </relocation>
-                <relocation>
-                  <pattern>com.github.luben</pattern>
-                  <shadedPattern>${shadeBase}.com.github.luben</shadedPattern>
-                </relocation>
                 <relocation>
                   <pattern>com.thoughtworks</pattern>
                   <shadedPattern>${shadeBase}.com.thoughtworks</shadedPattern>
diff --git a/public_pom.xml b/public_pom.xml
index 9ef308a08..fb03684cf 100644
--- a/public_pom.xml
+++ b/public_pom.xml
@@ -48,5 +48,11 @@
             <version>1.7.36</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>com.github.luben</groupId>
+            <artifactId>zstd-jni</artifactId>
+            <version>1.5.0-1</version>
+            <scope>runtime</scope>
+        </dependency>
     </dependencies>
 </project>
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index a22bc9f21..404ec3851 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -119,7 +119,8 @@ public static BdecVersion fromInt(int val) {
    * CompressionCodecName, but we want to control and allow only specific values of that.
    */
   public enum BdecParquetCompression {
-    GZIP;
+    GZIP,
+    ZSTD;
 
     public CompressionCodecName getCompressionCodec() {
       return CompressionCodecName.fromConf(this.name());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
index 1ee6b9542..a32b3a25d 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
@@ -308,6 +308,17 @@ public void testValidCompressionAlgorithmsAndWithUppercaseLowerCase() {
               Constants.BdecParquetCompression.GZIP,
               parameterProvider.getBdecParquetCompressionAlgorithm());
         });
+    List<String> zstdValues = Arrays.asList("ZSTD", "zstd", "Zstd", "zStd");
+    zstdValues.forEach(
+            v -> {
+              Properties prop = new Properties();
+              Map<String, Object> parameterMap = getStartingParameterMap();
+              parameterMap.put(ParameterProvider.BDEC_PARQUET_COMPRESSION_ALGORITHM, v);
+              ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+              Assert.assertEquals(
+                      Constants.BdecParquetCompression.ZSTD,
+                      parameterProvider.getBdecParquetCompressionAlgorithm());
+            });
   }
 
   @Test
@@ -322,7 +333,7 @@ public void testInvalidCompressionAlgorithm() {
     } catch (IllegalArgumentException e) {
       Assert.assertEquals(
           "Unsupported BDEC_PARQUET_COMPRESSION_ALGORITHM = 'invalid_comp', allowed values are"
-              + " [GZIP]",
+              + " [GZIP, ZSTD]",
           e.getMessage());
     }
   }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
index 7994ff7bf..62494037c 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
@@ -2,6 +2,7 @@
 
 import static net.snowflake.ingest.TestUtils.verifyTableRowCount;
 import static net.snowflake.ingest.utils.Constants.ROLE;
+import static net.snowflake.ingest.utils.ParameterProvider.BDEC_PARQUET_COMPRESSION_ALGORITHM;
 
 import java.sql.Connection;
 import java.sql.ResultSet;
@@ -17,8 +18,13 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
 
 /** Ingest large amount of rows. */
+@RunWith(Parameterized.class)
 public class StreamingIngestBigFilesIT {
   private static final String TEST_DB_PREFIX = "STREAMING_INGEST_TEST_DB";
   private static final String TEST_SCHEMA = "STREAMING_INGEST_TEST_SCHEMA";
@@ -29,6 +35,14 @@ public class StreamingIngestBigFilesIT {
   private Connection jdbcConnection;
   private String testDb;
 
+  @Parameters(name = "{index}: {0}")
+  public static Object[] compressionAlgorithms() {
+    return new Object[] {"GZIP", "ZSTD"};
+  }
+
+  @Parameter
+  public String compressionAlgorithm;
+
   @Before
   public void beforeAll() throws Exception {
     testDb = TEST_DB_PREFIX + "_" + UUID.randomUUID().toString().substring(0, 4);
@@ -51,6 +65,7 @@ public void beforeAll() throws Exception {
     if (prop.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       prop.setProperty(ROLE, "ACCOUNTADMIN");
     }
+    prop.setProperty(BDEC_PARQUET_COMPRESSION_ALGORITHM, compressionAlgorithm);
     client =
         (SnowflakeStreamingIngestClientInternal<?>)
             SnowflakeStreamingIngestClientFactory.builder("client1").setProperties(prop).build();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 3609b8e5e..38769e0b1 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -3,6 +3,7 @@
 import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
 import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
 import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
+import static net.snowflake.ingest.utils.ParameterProvider.BDEC_PARQUET_COMPRESSION_ALGORITHM;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.Is.is;
 
@@ -45,10 +46,15 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
 import org.mockito.ArgumentMatchers;
 import org.mockito.Mockito;
 
 /** Example streaming ingest sdk integration test */
+@RunWith(Parameterized.class)
 public class StreamingIngestIT {
   private static final String TEST_TABLE = "STREAMING_INGEST_TEST_TABLE";
   private static final String TEST_DB_PREFIX = "STREAMING_INGEST_TEST_DB";
@@ -64,6 +70,14 @@ public class StreamingIngestIT {
   private Connection jdbcConnection;
   private String testDb;
 
+  @Parameters(name = "{index}: {0}")
+  public static Object[] compressionAlgorithms() {
+    return new Object[] {"GZIP", "ZSTD"};
+  }
+
+  @Parameter
+  public String compressionAlgorithm;
+
   @Before
   public void beforeAll() throws Exception {
     testDb = TEST_DB_PREFIX + "_" + UUID.randomUUID().toString().substring(0, 4);
@@ -90,7 +104,7 @@ public void beforeAll() throws Exception {
 
     // Test without role param
     prop = TestUtils.getProperties(Constants.BdecVersion.THREE, true);
-
+    prop.setProperty(BDEC_PARQUET_COMPRESSION_ALGORITHM, compressionAlgorithm);
     client =
         (SnowflakeStreamingIngestClientInternal<?>)
             SnowflakeStreamingIngestClientFactory.builder("client1").setProperties(prop).build();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index f449be2d5..b78473a11 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -22,11 +22,17 @@
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
 import net.snowflake.ingest.utils.Constants;
+import static net.snowflake.ingest.utils.ParameterProvider.BDEC_PARQUET_COMPRESSION_ALGORITHM;
 import net.snowflake.ingest.utils.SFException;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
 
+@RunWith(Parameterized.class)
 public abstract class AbstractDataTypeTest {
   private static final String SOURCE_COLUMN_NAME = "source";
   private static final String VALUE_COLUMN_NAME = "value";
@@ -56,6 +62,14 @@ public abstract class AbstractDataTypeTest {
   private SnowflakeStreamingIngestClient client;
   private static final ObjectMapper objectMapper = new ObjectMapper();
 
+  @Parameters(name = "{index}: {0}")
+  public static Object[] compressionAlgorithms() {
+    return new Object[] {"GZIP", "ZSTD"};
+  }
+
+  @Parameter
+  public String compressionAlgorithm;
+
   @Before
   public void before() throws Exception {
     databaseName = String.format("SDK_DATATYPE_COMPATIBILITY_IT_%s", getRandomIdentifier());
@@ -70,6 +84,7 @@ public void before() throws Exception {
     if (props.getProperty(ROLE).equals("DEFAULT_ROLE")) {
       props.setProperty(ROLE, "ACCOUNTADMIN");
     }
+    props.setProperty(BDEC_PARQUET_COMPRESSION_ALGORITHM, compressionAlgorithm);
     client = SnowflakeStreamingIngestClientFactory.builder("client1").setProperties(props).build();
   }
 

From 1dc0ce0d231b63043ab69f22a668840110be122d Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Mon, 5 Feb 2024 12:25:04 -0800
Subject: [PATCH 319/356] SNOW-1032874: Fix an Overflow Issue for Integer-store
 Timestamp  (#673)

According to https://docs.snowflake.com/en/sql-reference/functions/to_timestamp#usage-notes:

- If the integer is less than 31536000000 (the number of milliseconds in a year), then the value is treated as a number of seconds.
- If the value is greater than or equal to 31536000000 and less than 31536000000000, then the value is treated as milliseconds.
- If the value is greater than or equal to 31536000000000 and less than 31536000000000000, then the value is treated as microseconds.
- If the value is greater than or equal to 31536000000000000, then the value is treated as nanoseconds.

Due to the multiplication logic in the code, we run into an overflow issue when the input is bigger than a certain value (for example: 31535999999999) and it causes silent corrupted values in the table, this change updates the code to use BigInteger instead of long to resolve the overflow issue.
---
 .../internal/DataValidationUtil.java          | 31 +++++++++-------
 .../streaming/internal/TimestampWrapper.java  |  2 +-
 .../internal/DataValidationUtilTest.java      | 35 +++++++++++++++++--
 .../datatypes/AbstractDataTypeTest.java       |  4 ++-
 .../internal/datatypes/DateTimeIT.java        | 34 +++++++++++++++++-
 5 files changed, 88 insertions(+), 18 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
index a1831f829..814423c28 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java
@@ -728,20 +728,27 @@ static BigInteger validateAndParseTime(
    * @throws NumberFormatException If the input in not a valid long
    */
   private static Instant parseInstantGuessScale(String input) {
-    long epochNanos;
-    long val = Long.parseLong(input);
-
-    if (val > -SECONDS_LIMIT_FOR_EPOCH && val < SECONDS_LIMIT_FOR_EPOCH) {
-      epochNanos = val * Power10.intTable[9];
-    } else if (val > -MILLISECONDS_LIMIT_FOR_EPOCH && val < MILLISECONDS_LIMIT_FOR_EPOCH) {
-      epochNanos = val * Power10.intTable[6];
-    } else if (val > -MICROSECONDS_LIMIT_FOR_EPOCH && val < MICROSECONDS_LIMIT_FOR_EPOCH) {
-      epochNanos = val * Power10.intTable[3];
-    } else {
-      epochNanos = val;
+    BigInteger epochNanos;
+    try {
+      long val = Long.parseLong(input);
+
+      if (val > -SECONDS_LIMIT_FOR_EPOCH && val < SECONDS_LIMIT_FOR_EPOCH) {
+        epochNanos = BigInteger.valueOf(val).multiply(Power10.sb16Table[9]);
+      } else if (val > -MILLISECONDS_LIMIT_FOR_EPOCH && val < MILLISECONDS_LIMIT_FOR_EPOCH) {
+        epochNanos = BigInteger.valueOf(val).multiply(Power10.sb16Table[6]);
+      } else if (val > -MICROSECONDS_LIMIT_FOR_EPOCH && val < MICROSECONDS_LIMIT_FOR_EPOCH) {
+        epochNanos = BigInteger.valueOf(val).multiply(Power10.sb16Table[3]);
+      } else {
+        epochNanos = BigInteger.valueOf(val);
+      }
+    } catch (NumberFormatException e) {
+      // The input is bigger than max long value, treat it as nano-seconds directly
+      epochNanos = new BigInteger(input);
     }
+
     return Instant.ofEpochSecond(
-        epochNanos / Power10.intTable[9], epochNanos % Power10.intTable[9]);
+        epochNanos.divide(Power10.sb16Table[9]).longValue(),
+        epochNanos.remainder(Power10.sb16Table[9]).longValue());
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/TimestampWrapper.java b/src/main/java/net/snowflake/ingest/streaming/internal/TimestampWrapper.java
index 871cf8c14..a2dc55570 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/TimestampWrapper.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/TimestampWrapper.java
@@ -74,7 +74,7 @@ public BigInteger toBinary(boolean includeTimezone) {
   }
 
   /** Get epoch in seconds */
-  public long getEpoch() {
+  public long getEpochSecond() {
     return epoch;
   }
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 86706fcf2..9467899e2 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -26,6 +26,8 @@
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
@@ -251,20 +253,47 @@ public void testValidateAndParseTime() {
   }
 
   @Test
-  public void testValidateAndParseTimestamp() {
+  public void testValidateAndParseTimestamp() throws ParseException {
     TimestampWrapper wrapper =
         DataValidationUtil.validateAndParseTimestamp(
             "COL", "2021-01-01T01:00:00.123+01:00", 4, UTC, false, 0);
-    assertEquals(1609459200, wrapper.getEpoch());
+    assertEquals(1609459200, wrapper.getEpochSecond());
     assertEquals(123000000, wrapper.getFraction());
     assertEquals(3600, wrapper.getTimezoneOffsetSeconds());
     assertEquals(1500, wrapper.getTimeZoneIndex());
 
     wrapper = validateAndParseTimestamp("COL", "  2021-01-01T01:00:00.123 \t\n", 9, UTC, true, 0);
-    Assert.assertEquals(1609462800, wrapper.getEpoch());
+    Assert.assertEquals(1609462800, wrapper.getEpochSecond());
     Assert.assertEquals(123000000, wrapper.getFraction());
     Assert.assertEquals(new BigInteger("1609462800123000000"), wrapper.toBinary(false));
 
+    // Test integer-stored time and scale guessing
+    SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
+    assertEquals(
+        BigInteger.valueOf(df.parse("1971-01-01 00:00:00.001").getTime())
+            .multiply(BigInteger.valueOf(1000000)),
+        validateAndParseTimestamp("COL", "31536000001", 9, UTC, true, 0).toBinary(false));
+
+    assertEquals(
+        BigInteger.valueOf(df.parse("2969-05-02 23:59:59.999").getTime())
+            .multiply(BigInteger.valueOf(1000000)),
+        validateAndParseTimestamp("COL", "31535999999999", 9, UTC, true, 0).toBinary(false));
+
+    assertEquals(
+        BigInteger.valueOf(df.parse("1971-01-01 00:00:00.000").getTime())
+            .multiply(BigInteger.valueOf(1000000)),
+        validateAndParseTimestamp("COL", "31536000000000", 9, UTC, true, 0).toBinary(false));
+
+    assertEquals(
+        BigInteger.valueOf(df.parse("2969-05-02 23:59:59.999").getTime())
+            .multiply(BigInteger.valueOf(1000000)),
+        validateAndParseTimestamp("COL", "31535999999999", 9, UTC, true, 0).toBinary(false));
+
+    assertEquals(
+        BigInteger.valueOf(df.parse("1971-01-01 00:00:00.000").getTime())
+            .multiply(BigInteger.valueOf(1000000)),
+        validateAndParseTimestamp("COL", "31536000000000000", 9, UTC, true, 0).toBinary(false));
+
     // Time input is not supported
     expectError(
         ErrorCode.INVALID_VALUE_ROW,
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index b78473a11..6718a0aeb 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -91,7 +91,9 @@ public void before() throws Exception {
   @After
   public void after() throws Exception {
     this.defaultTimezone = Optional.empty();
-    conn.createStatement().executeQuery(String.format("drop database %s", databaseName));
+    if (conn != null) {
+      conn.createStatement().executeQuery(String.format("drop database %s", databaseName));
+    }
     if (client != null) {
       client.close();
     }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
index 9bb8f349a..62829656c 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/DateTimeIT.java
@@ -27,7 +27,9 @@ public void setup() throws Exception {
 
   @After
   public void tearDown() throws Exception {
-    conn.createStatement().execute("alter session unset timezone;");
+    if (conn != null) {
+      conn.createStatement().execute("alter session unset timezone;");
+    }
   }
 
   @Test
@@ -639,6 +641,36 @@ public void testTimestampWithoutTimeZone() throws Exception {
         "1916-12-09 10:57:53.876543211 Z",
         new StringProvider(),
         new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "31536000001",
+        "1971-01-01 00:00:00.001000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "31535999999999",
+        "2969-05-02 23:59:59.999000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "31536000000000",
+        "1971-01-01 00:00:00.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "31535999999999",
+        "2969-05-02 23:59:59.999000000 Z",
+        new StringProvider(),
+        new StringProvider());
+    testJdbcTypeCompatibility(
+        "TIMESTAMP_NTZ",
+        "31536000000000000",
+        "1971-01-01 00:00:00.000000000 Z",
+        new StringProvider(),
+        new StringProvider());
 
     // java.time.LocalDate
     testIngestion(

From 38badbb6e4c1f44b01fda77a218e00629657f387 Mon Sep 17 00:00:00 2001
From: Andrey Zagrebin <andrey.zagrebin@snowflake.com>
Date: Wed, 7 Feb 2024 15:23:52 +0000
Subject: [PATCH 320/356] Snowpipe Streaming: send column ordinal back to GS
 with blob EPs to register

---
 .../internal/FileColumnProperties.java        | 17 ++++++++--
 .../streaming/internal/ParquetRowBuffer.java  |  4 +--
 .../streaming/internal/RowBufferStats.java    | 14 +++++---
 .../internal/FileColumnPropertiesTest.java    |  6 ++--
 .../streaming/internal/FlushServiceTest.java  |  3 ++
 .../streaming/internal/RowBufferTest.java     | 34 +++++++++++++++++--
 .../SnowflakeStreamingIngestChannelTest.java  |  8 +++--
 7 files changed, 72 insertions(+), 14 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java b/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java
index 290248a83..97fd0b712 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java
@@ -8,7 +8,7 @@
 
 /** Audit register endpoint/FileColumnPropertyDTO property list. */
 class FileColumnProperties {
-
+  private int columnOrdinal;
   private String minStrValue;
 
   private String maxStrValue;
@@ -45,6 +45,7 @@ class FileColumnProperties {
   public static final Double DEFAULT_MIN_MAX_REAL_VAL_FOR_EP = 0d;
 
   FileColumnProperties(RowBufferStats stats) {
+    this.setColumnOrdinal(stats.getOrdinal());
     this.setCollation(stats.getCollationDefinitionString());
     this.setMaxIntValue(
         stats.getCurrentMaxIntValue() == null
@@ -83,6 +84,15 @@ class FileColumnProperties {
     this.setDistinctValues(stats.getDistinctValues());
   }
 
+  @JsonProperty("columnId")
+  public int getColumnOrdinal() {
+    return columnOrdinal;
+  }
+
+  public void setColumnOrdinal(int columnOrdinal) {
+    this.columnOrdinal = columnOrdinal;
+  }
+
   // Annotation required in order to have package private fields serialized
   @JsonProperty("minStrValue")
   String getMinStrValue() {
@@ -195,6 +205,7 @@ void setMaxStrNonCollated(String maxStrNonCollated) {
   @Override
   public String toString() {
     final StringBuilder sb = new StringBuilder("{");
+    sb.append("\"columnOrdinal\": ").append(columnOrdinal);
     if (minIntValue != null) {
       sb.append(", \"minIntValue\": ").append(minIntValue);
       sb.append(", \"maxIntValue\": ").append(maxIntValue);
@@ -220,7 +231,8 @@ public boolean equals(Object o) {
     if (this == o) return true;
     if (o == null || getClass() != o.getClass()) return false;
     FileColumnProperties that = (FileColumnProperties) o;
-    return distinctValues == that.distinctValues
+    return Objects.equals(columnOrdinal,that.columnOrdinal)
+        && distinctValues == that.distinctValues
         && nullCount == that.nullCount
         && maxLength == that.maxLength
         && Objects.equals(minStrValue, that.minStrValue)
@@ -237,6 +249,7 @@ public boolean equals(Object o) {
   @Override
   public int hashCode() {
     return Objects.hash(
+        columnOrdinal,
         minStrValue,
         maxStrValue,
         collation,
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 602e34400..c587d52aa 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -95,12 +95,12 @@ public void setupSchema(List<ColumnMetadata> columns) {
         addNonNullableFieldName(column.getInternalName());
       }
       this.statsMap.put(
-          column.getInternalName(), new RowBufferStats(column.getName(), column.getCollation()));
+          column.getInternalName(), new RowBufferStats(column.getName(), column.getCollation(), column.getOrdinal()));
 
       if (onErrorOption == OpenChannelRequest.OnErrorOption.ABORT
           || onErrorOption == OpenChannelRequest.OnErrorOption.SKIP_BATCH) {
         this.tempStatsMap.put(
-            column.getInternalName(), new RowBufferStats(column.getName(), column.getCollation()));
+            column.getInternalName(), new RowBufferStats(column.getName(), column.getCollation(), column.getOrdinal()));
       }
 
       id++;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
index 88cc568d7..840368764 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
@@ -15,6 +15,7 @@
 /** Keeps track of the active EP stats, used to generate a file EP info */
 class RowBufferStats {
 
+  private final int ordinal;
   private byte[] currentMinStrValue;
   private byte[] currentMaxStrValue;
   private BigInteger currentMinIntValue;
@@ -29,14 +30,15 @@ class RowBufferStats {
   private final String columnDisplayName;
 
   /** Creates empty stats */
-  RowBufferStats(String columnDisplayName, String collationDefinitionString) {
+  RowBufferStats(String columnDisplayName, String collationDefinitionString, int ordinal) {
     this.columnDisplayName = columnDisplayName;
     this.collationDefinitionString = collationDefinitionString;
+    this.ordinal= ordinal;
     reset();
   }
 
   RowBufferStats(String columnDisplayName) {
-    this(columnDisplayName, null);
+    this(columnDisplayName, null, -1);
   }
 
   void reset() {
@@ -52,7 +54,7 @@ void reset() {
 
   /** Create new statistics for the same column, with all calculated values set to empty */
   RowBufferStats forkEmpty() {
-    return new RowBufferStats(this.getColumnDisplayName(), this.getCollationDefinitionString());
+    return new RowBufferStats(this.getColumnDisplayName(), this.getCollationDefinitionString(), this.getOrdinal());
   }
 
   // TODO performance test this vs in place update
@@ -66,7 +68,7 @@ static RowBufferStats getCombinedStats(RowBufferStats left, RowBufferStats right
               left.getCollationDefinitionString(), right.getCollationDefinitionString()));
     }
     RowBufferStats combined =
-        new RowBufferStats(left.columnDisplayName, left.getCollationDefinitionString());
+        new RowBufferStats(left.columnDisplayName, left.getCollationDefinitionString(), left.getOrdinal());
 
     if (left.currentMinIntValue != null) {
       combined.addIntValue(left.currentMinIntValue);
@@ -209,6 +211,10 @@ String getColumnDisplayName() {
     return columnDisplayName;
   }
 
+  public int getOrdinal() {
+    return ordinal;
+  }
+
   /**
    * Compares two byte arrays lexicographically. If the two arrays share a common prefix then the
    * lexicographic comparison is the result of comparing two elements, as if by Byte.compare(byte,
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FileColumnPropertiesTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FileColumnPropertiesTest.java
index e056f4ab6..11e128ef7 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FileColumnPropertiesTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FileColumnPropertiesTest.java
@@ -8,20 +8,22 @@ public class FileColumnPropertiesTest {
   @Test
   public void testFileColumnPropertiesConstructor() {
     // Test simple construction
-    RowBufferStats stats = new RowBufferStats("COL");
+    RowBufferStats stats = new RowBufferStats("COL", null, 1);
     stats.addStrValue("bcd");
     stats.addStrValue("abcde");
     FileColumnProperties props = new FileColumnProperties(stats);
+    Assert.assertEquals(1, props.getColumnOrdinal());
     Assert.assertEquals("6162636465", props.getMinStrValue());
     Assert.assertNull(props.getMinStrNonCollated());
     Assert.assertEquals("626364", props.getMaxStrValue());
     Assert.assertNull(props.getMaxStrNonCollated());
 
     // Test that truncation is performed
-    stats = new RowBufferStats("COL");
+    stats = new RowBufferStats("COL", null, 1);
     stats.addStrValue("aßßßßßßßßßßßßßßßß");
     Assert.assertEquals(33, stats.getCurrentMinStrValue().length);
     props = new FileColumnProperties(stats);
+    Assert.assertEquals(1, props.getColumnOrdinal());
     Assert.assertNull(props.getMinStrNonCollated());
     Assert.assertNull(props.getMaxStrNonCollated());
     Assert.assertEquals(32 * 2, props.getMinStrValue().length());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index 17f929206..c756964ca 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -349,6 +349,7 @@ private SnowflakeStreamingIngestChannelInternal<?> addChannel4(TestContext<?> te
 
   private static ColumnMetadata createTestIntegerColumn(String name) {
     ColumnMetadata colInt = new ColumnMetadata();
+    colInt.setOrdinal(1);
     colInt.setName(name);
     colInt.setPhysicalType("SB4");
     colInt.setNullable(true);
@@ -360,6 +361,7 @@ private static ColumnMetadata createTestIntegerColumn(String name) {
 
   private static ColumnMetadata createTestTextColumn(String name) {
     ColumnMetadata colChar = new ColumnMetadata();
+    colChar.setOrdinal(1);
     colChar.setName(name);
     colChar.setPhysicalType("LOB");
     colChar.setNullable(true);
@@ -372,6 +374,7 @@ private static ColumnMetadata createTestTextColumn(String name) {
 
   private static ColumnMetadata createLargeTestTextColumn(String name) {
     ColumnMetadata colChar = new ColumnMetadata();
+    colChar.setOrdinal(1);
     colChar.setName(name);
     colChar.setPhysicalType("LOB");
     colChar.setNullable(true);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 52d8eed2e..f26c999b7 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -20,6 +20,7 @@
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 import org.apache.commons.codec.binary.Hex;
+import org.checkerframework.common.value.qual.IntRange;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -104,8 +105,12 @@ static List<ColumnMetadata> createSchema() {
     colChar.setLength(11);
     colChar.setScale(0);
 
-    return Arrays.asList(
-        colTinyIntCase, colTinyInt, colSmallInt, colInt, colBigInt, colDecimal, colChar);
+    List<ColumnMetadata> columns = Arrays.asList(
+            colTinyIntCase, colTinyInt, colSmallInt, colInt, colBigInt, colDecimal, colChar);
+    for (int i = 0; i < columns.size(); i++) {
+      columns.get(i).setOrdinal(i + 1);
+    }
+    return columns;
   }
 
   private AbstractRowBuffer<?> createTestBuffer(OpenChannelRequest.OnErrorOption onErrorOption) {
@@ -506,6 +511,7 @@ private void testDoubleQuotesColumnNameHelper(OpenChannelRequest.OnErrorOption o
     AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colDoubleQuotes = new ColumnMetadata();
+    colDoubleQuotes.setOrdinal(1);
     colDoubleQuotes.setName("\"colDoubleQuotes\"");
     colDoubleQuotes.setPhysicalType("SB16");
     colDoubleQuotes.setNullable(true);
@@ -662,6 +668,7 @@ public void testE2ETimestampErrors() {
   private void testE2ETimestampErrorsHelper(AbstractRowBuffer<?> innerBuffer) {
 
     ColumnMetadata colTimestampLtzSB16 = new ColumnMetadata();
+    colTimestampLtzSB16.setOrdinal(1);
     colTimestampLtzSB16.setName("COLTIMESTAMPLTZ_SB16");
     colTimestampLtzSB16.setPhysicalType("SB16");
     colTimestampLtzSB16.setNullable(false);
@@ -782,6 +789,7 @@ private void testStatsE2ETimestampHelper(OpenChannelRequest.OnErrorOption onErro
     AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colTimestampLtzSB8 = new ColumnMetadata();
+    colTimestampLtzSB8.setOrdinal(1);
     colTimestampLtzSB8.setName("COLTIMESTAMPLTZ_SB8");
     colTimestampLtzSB8.setPhysicalType("SB8");
     colTimestampLtzSB8.setNullable(true);
@@ -789,6 +797,7 @@ private void testStatsE2ETimestampHelper(OpenChannelRequest.OnErrorOption onErro
     colTimestampLtzSB8.setScale(0);
 
     ColumnMetadata colTimestampLtzSB16 = new ColumnMetadata();
+    colTimestampLtzSB16.setOrdinal(2);
     colTimestampLtzSB16.setName("COLTIMESTAMPLTZ_SB16");
     colTimestampLtzSB16.setPhysicalType("SB16");
     colTimestampLtzSB16.setNullable(true);
@@ -796,6 +805,7 @@ private void testStatsE2ETimestampHelper(OpenChannelRequest.OnErrorOption onErro
     colTimestampLtzSB16.setScale(9);
 
     ColumnMetadata colTimestampLtzSB16Scale6 = new ColumnMetadata();
+    colTimestampLtzSB16Scale6.setOrdinal(2);
     colTimestampLtzSB16Scale6.setName("COLTIMESTAMPLTZ_SB16_SCALE6");
     colTimestampLtzSB16Scale6.setPhysicalType("SB16");
     colTimestampLtzSB16Scale6.setNullable(true);
@@ -864,6 +874,7 @@ private void testE2EDateHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
     AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colDate = new ColumnMetadata();
+    colDate.setOrdinal(1);
     colDate.setName("COLDATE");
     colDate.setPhysicalType("SB8");
     colDate.setNullable(true);
@@ -913,6 +924,7 @@ private void testE2ETimeHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
     AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colTimeSB4 = new ColumnMetadata();
+    colTimeSB4.setOrdinal(1);
     colTimeSB4.setName("COLTIMESB4");
     colTimeSB4.setPhysicalType("SB4");
     colTimeSB4.setNullable(true);
@@ -920,6 +932,7 @@ private void testE2ETimeHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
     colTimeSB4.setScale(0);
 
     ColumnMetadata colTimeSB8 = new ColumnMetadata();
+    colTimeSB8.setOrdinal(2);
     colTimeSB8.setName("COLTIMESB8");
     colTimeSB8.setPhysicalType("SB8");
     colTimeSB8.setNullable(true);
@@ -984,6 +997,7 @@ public void testMaxInsertRowsBatchSize() {
   private void testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
     AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
     ColumnMetadata colBinary = new ColumnMetadata();
+    colBinary.setOrdinal(1);
     colBinary.setName("COLBINARY");
     colBinary.setPhysicalType("LOB");
     colBinary.setNullable(true);
@@ -1023,6 +1037,7 @@ private void testNullableCheckHelper(OpenChannelRequest.OnErrorOption onErrorOpt
     AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colBoolean = new ColumnMetadata();
+    colBoolean.setOrdinal(1);
     colBoolean.setName("COLBOOLEAN");
     colBoolean.setPhysicalType("SB1");
     colBoolean.setNullable(false);
@@ -1063,6 +1078,7 @@ private void testMissingColumnCheckHelper(OpenChannelRequest.OnErrorOption onErr
     AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colBoolean = new ColumnMetadata();
+    colBoolean.setOrdinal(1);
     colBoolean.setName("COLBOOLEAN");
     colBoolean.setPhysicalType("SB1");
     colBoolean.setNullable(false);
@@ -1070,6 +1086,7 @@ private void testMissingColumnCheckHelper(OpenChannelRequest.OnErrorOption onErr
     colBoolean.setScale(0);
 
     ColumnMetadata colBoolean2 = new ColumnMetadata();
+    colBoolean2.setOrdinal(2);
     colBoolean2.setName("COLBOOLEAN2");
     colBoolean2.setPhysicalType("SB1");
     colBoolean2.setNullable(true);
@@ -1107,6 +1124,7 @@ public void testExtraColumnsCheck() {
     AbstractRowBuffer<?> innerBuffer = createTestBuffer(OpenChannelRequest.OnErrorOption.CONTINUE);
 
     ColumnMetadata colBoolean = new ColumnMetadata();
+    colBoolean.setOrdinal(1);
     colBoolean.setName("COLBOOLEAN1");
     colBoolean.setPhysicalType("SB1");
     colBoolean.setNullable(false);
@@ -1139,6 +1157,7 @@ private void doTestFailureHalfwayThroughColumnProcessing(
     AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colVarchar1 = new ColumnMetadata();
+    colVarchar1.setOrdinal(1);
     colVarchar1.setName("COLVARCHAR1");
     colVarchar1.setPhysicalType("LOB");
     colVarchar1.setNullable(true);
@@ -1146,6 +1165,7 @@ private void doTestFailureHalfwayThroughColumnProcessing(
     colVarchar1.setLength(1000);
 
     ColumnMetadata colVarchar2 = new ColumnMetadata();
+    colVarchar2.setOrdinal(2);
     colVarchar2.setName("COLVARCHAR2");
     colVarchar2.setPhysicalType("LOB");
     colVarchar2.setNullable(true);
@@ -1153,6 +1173,7 @@ private void doTestFailureHalfwayThroughColumnProcessing(
     colVarchar2.setLength(1000);
 
     ColumnMetadata colBoolean = new ColumnMetadata();
+    colBoolean.setOrdinal(3);
     colBoolean.setName("COLBOOLEAN1");
     colBoolean.setPhysicalType("SB1");
     colBoolean.setNullable(true);
@@ -1215,6 +1236,7 @@ private void testE2EBooleanHelper(OpenChannelRequest.OnErrorOption onErrorOption
     AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colBoolean = new ColumnMetadata();
+    colBoolean.setOrdinal(1);
     colBoolean.setName("COLBOOLEAN");
     colBoolean.setPhysicalType("SB1");
     colBoolean.setNullable(true);
@@ -1264,6 +1286,7 @@ private void testE2EBinaryHelper(OpenChannelRequest.OnErrorOption onErrorOption)
     AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colBinary = new ColumnMetadata();
+    colBinary.setOrdinal(1);
     colBinary.setName("COLBINARY");
     colBinary.setPhysicalType("LOB");
     colBinary.setNullable(true);
@@ -1321,6 +1344,7 @@ private void testE2ERealHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
     AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colReal = new ColumnMetadata();
+    colReal.setOrdinal(1);
     colReal.setName("COLREAL");
     colReal.setPhysicalType("SB16");
     colReal.setNullable(true);
@@ -1363,6 +1387,7 @@ public void testOnErrorAbortFailures() {
     AbstractRowBuffer<?> innerBuffer = createTestBuffer(OpenChannelRequest.OnErrorOption.ABORT);
 
     ColumnMetadata colDecimal = new ColumnMetadata();
+    colDecimal.setOrdinal(1);
     colDecimal.setName("COLDECIMAL");
     colDecimal.setPhysicalType("SB16");
     colDecimal.setNullable(true);
@@ -1440,6 +1465,7 @@ public void testOnErrorAbortSkipBatch() {
         createTestBuffer(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
 
     ColumnMetadata colDecimal = new ColumnMetadata();
+    colDecimal.setOrdinal(1);
     colDecimal.setName("COLDECIMAL");
     colDecimal.setPhysicalType("SB16");
     colDecimal.setNullable(true);
@@ -1517,6 +1543,7 @@ private void testE2EVariantHelper(OpenChannelRequest.OnErrorOption onErrorOption
     AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colVariant = new ColumnMetadata();
+    colVariant.setOrdinal(1);
     colVariant.setName("COLVARIANT");
     colVariant.setPhysicalType("LOB");
     colVariant.setNullable(true);
@@ -1567,6 +1594,7 @@ private void testE2EObjectHelper(OpenChannelRequest.OnErrorOption onErrorOption)
     AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colObject = new ColumnMetadata();
+    colObject.setOrdinal(1);
     colObject.setName("COLOBJECT");
     colObject.setPhysicalType("LOB");
     colObject.setNullable(true);
@@ -1599,6 +1627,7 @@ private void testE2EArrayHelper(OpenChannelRequest.OnErrorOption onErrorOption)
     AbstractRowBuffer<?> innerBuffer = createTestBuffer(onErrorOption);
 
     ColumnMetadata colObject = new ColumnMetadata();
+    colObject.setOrdinal(1);
     colObject.setName("COLARRAY");
     colObject.setPhysicalType("LOB");
     colObject.setNullable(true);
@@ -1647,6 +1676,7 @@ public void testOnErrorAbortRowsWithError() {
         createTestBuffer(OpenChannelRequest.OnErrorOption.SKIP_BATCH);
 
     ColumnMetadata colChar = new ColumnMetadata();
+    colChar.setOrdinal(1);
     colChar.setName("COLCHAR");
     colChar.setPhysicalType("LOB");
     colChar.setNullable(true);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 528f9e46d..d419fffc0 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -455,6 +455,7 @@ public void testOpenChannelSuccessResponse() throws Exception {
             + "  \"offset_token\" : \"\",\n"
             + "  \"encryption_key_id\" : 17229585102,\n"
             + "  \"table_columns\" : [ {\n"
+            + "    \"ordinal\" : 1,\n"
             + "    \"name\" : \"C1\",\n"
             + "    \"type\" : \"NUMBER(38,0)\",\n"
             + "    \"logical_type\" : \"fixed\",\n"
@@ -465,6 +466,7 @@ public void testOpenChannelSuccessResponse() throws Exception {
             + "    \"length\" : null,\n"
             + "    \"nullable\" : true\n"
             + "  }, {\n"
+            + "    \"ordinal\" : 2,\n"
             + "    \"name\" : \"C2\",\n"
             + "    \"type\" : \"NUMBER(38,0)\",\n"
             + "    \"logical_type\" : \"fixed\",\n"
@@ -551,6 +553,7 @@ public void testInsertRow() {
             UTC);
 
     ColumnMetadata col = new ColumnMetadata();
+    col.setOrdinal(1);
     col.setName("COL");
     col.setPhysicalType("SB16");
     col.setNullable(false);
@@ -593,9 +596,10 @@ public void testInsertTooLargeRow() {
     List<ColumnMetadata> schema =
         IntStream.range(0, 64)
             .mapToObj(
-                rowId -> {
+                colId -> {
                   ColumnMetadata col = new ColumnMetadata();
-                  col.setName("COL" + rowId);
+                  col.setOrdinal(colId + 1);
+                  col.setName("COL" + colId);
                   col.setPhysicalType("LOB");
                   col.setNullable(false);
                   col.setLogicalType("BINARY");

From f33444099885c661188264eea648bb021c0870a0 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Fri, 16 Feb 2024 00:37:51 -0800
Subject: [PATCH 321/356] SNOW-979849: Add start offset token at batch level in
 the Client SDK (#676)

Support start offset token to be part of the interface so that it can be used to log and verify offset related behavior in upcoming PRs
---
 .../SnowflakeStreamingIngestChannel.java      |  14 ++-
 .../streaming/internal/AbstractRowBuffer.java |  37 ++++--
 .../internal/ChannelRuntimeState.java         |  34 ++++--
 .../streaming/internal/IngestionStrategy.java |   8 +-
 .../streaming/internal/ParquetRowBuffer.java  |   4 +-
 .../ingest/streaming/internal/RowBuffer.java  |   6 +-
 ...owflakeStreamingIngestChannelInternal.java |  34 ++++--
 ...nowflakeStreamingIngestClientInternal.java |   6 +-
 .../internal/ParameterProviderTest.java       |  18 +--
 .../streaming/internal/RowBufferTest.java     |  95 ++++++++-------
 .../SnowflakeStreamingIngestChannelTest.java  |   8 +-
 .../SnowflakeStreamingIngestClientTest.java   |  10 +-
 .../internal/StreamingIngestBigFilesIT.java   |   3 +-
 .../streaming/internal/StreamingIngestIT.java | 113 +++++++++++++++++-
 .../datatypes/AbstractDataTypeTest.java       |   5 +-
 15 files changed, 286 insertions(+), 109 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
index bb372c64c..2ab48b53f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
@@ -240,10 +240,20 @@ public interface SnowflakeStreamingIngestChannel {
    * values.
    *
    * @param rows object data to write
-   * @param offsetToken offset of last row in the row-set, used for replay in case of failures, It
-   *     could be null if you don't plan on replaying or can't replay
+   * @param startOffsetToken start offset of the batch/row-set
+   * @param endOffsetToken end offset of the batch/row-set, used for replay in case of failures, *
+   *     It could be null if you don't plan on replaying or can't replay
    * @return insert response that possibly contains errors because of insertion failures
    */
+  InsertValidationResponse insertRows(
+      Iterable<Map<String, Object>> rows,
+      @Nullable String startOffsetToken,
+      @Nullable String endOffsetToken);
+
+  /**
+   * Insert a batch of rows into the channel with the end offset token only, please see {@link
+   * SnowflakeStreamingIngestChannel#insertRows(Iterable, String, String)} for more information.
+   */
   InsertValidationResponse insertRows(
       Iterable<Map<String, Object>> rows, @Nullable String offsetToken);
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 766a24763..e111266e9 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -142,10 +142,14 @@ public int getOrdinal() {
   public class ContinueIngestionStrategy<T> implements IngestionStrategy<T> {
     @Override
     public InsertValidationResponse insertRows(
-        AbstractRowBuffer<T> rowBuffer, Iterable<Map<String, Object>> rows, String offsetToken) {
+        AbstractRowBuffer<T> rowBuffer,
+        Iterable<Map<String, Object>> rows,
+        String startOffsetToken,
+        String endOffsetToken) {
       InsertValidationResponse response = new InsertValidationResponse();
       float rowsSizeInBytes = 0F;
       int rowIndex = 0;
+      int prevRowCount = rowBuffer.bufferedRowCount;
       for (Map<String, Object> row : rows) {
         InsertValidationResponse.InsertError error =
             new InsertValidationResponse.InsertError(row, rowIndex);
@@ -169,7 +173,7 @@ public InsertValidationResponse insertRows(
         rowIndex++;
       }
       checkBatchSizeRecommendedMaximum(rowsSizeInBytes);
-      rowBuffer.channelState.setOffsetToken(offsetToken);
+      rowBuffer.channelState.updateOffsetToken(startOffsetToken, endOffsetToken, prevRowCount);
       rowBuffer.bufferSize += rowsSizeInBytes;
       rowBuffer.rowSizeMetric.accept(rowsSizeInBytes);
       return response;
@@ -180,7 +184,10 @@ public InsertValidationResponse insertRows(
   public class AbortIngestionStrategy<T> implements IngestionStrategy<T> {
     @Override
     public InsertValidationResponse insertRows(
-        AbstractRowBuffer<T> rowBuffer, Iterable<Map<String, Object>> rows, String offsetToken) {
+        AbstractRowBuffer<T> rowBuffer,
+        Iterable<Map<String, Object>> rows,
+        String startOffsetToken,
+        String endOffsetToken) {
       // If the on_error option is ABORT, simply throw the first exception
       InsertValidationResponse response = new InsertValidationResponse();
       float rowsSizeInBytes = 0F;
@@ -201,13 +208,14 @@ public InsertValidationResponse insertRows(
       moveTempRowsToActualBuffer(tempRowCount);
 
       rowsSizeInBytes = tempRowsSizeInBytes;
-      rowBuffer.bufferedRowCount += tempRowCount;
       rowBuffer.statsMap.forEach(
           (colName, stats) ->
               rowBuffer.statsMap.put(
                   colName,
                   RowBufferStats.getCombinedStats(stats, rowBuffer.tempStatsMap.get(colName))));
-      rowBuffer.channelState.setOffsetToken(offsetToken);
+      rowBuffer.channelState.updateOffsetToken(
+          startOffsetToken, endOffsetToken, rowBuffer.bufferedRowCount);
+      rowBuffer.bufferedRowCount += tempRowCount;
       rowBuffer.bufferSize += rowsSizeInBytes;
       rowBuffer.rowSizeMetric.accept(rowsSizeInBytes);
       return response;
@@ -218,7 +226,10 @@ public InsertValidationResponse insertRows(
   public class SkipBatchIngestionStrategy<T> implements IngestionStrategy<T> {
     @Override
     public InsertValidationResponse insertRows(
-        AbstractRowBuffer<T> rowBuffer, Iterable<Map<String, Object>> rows, String offsetToken) {
+        AbstractRowBuffer<T> rowBuffer,
+        Iterable<Map<String, Object>> rows,
+        String startOffsetToken,
+        String endOffsetToken) {
       InsertValidationResponse response = new InsertValidationResponse();
       float rowsSizeInBytes = 0F;
       float tempRowsSizeInBytes = 0F;
@@ -251,13 +262,14 @@ public InsertValidationResponse insertRows(
         checkBatchSizeRecommendedMaximum(tempRowsSizeInBytes);
         moveTempRowsToActualBuffer(tempRowCount);
         rowsSizeInBytes = tempRowsSizeInBytes;
-        rowBuffer.bufferedRowCount += tempRowCount;
         rowBuffer.statsMap.forEach(
             (colName, stats) ->
                 rowBuffer.statsMap.put(
                     colName,
                     RowBufferStats.getCombinedStats(stats, rowBuffer.tempStatsMap.get(colName))));
-        rowBuffer.channelState.setOffsetToken(offsetToken);
+        rowBuffer.channelState.updateOffsetToken(
+            startOffsetToken, endOffsetToken, rowBuffer.bufferedRowCount);
+        rowBuffer.bufferedRowCount += tempRowCount;
         rowBuffer.bufferSize += rowsSizeInBytes;
         rowBuffer.rowSizeMetric.accept(rowsSizeInBytes);
       }
@@ -418,12 +430,13 @@ Set<String> verifyInputColumns(
    * Insert a batch of rows into the row buffer
    *
    * @param rows input row
-   * @param offsetToken offset token of the latest row in the batch
+   * @param startOffsetToken start offset token of the batch
+   * @param endOffsetToken offset token of the latest row in the batch
    * @return insert response that possibly contains errors because of insertion failures
    */
   @Override
   public InsertValidationResponse insertRows(
-      Iterable<Map<String, Object>> rows, String offsetToken) {
+      Iterable<Map<String, Object>> rows, String startOffsetToken, String endOffsetToken) {
     if (!hasColumns()) {
       throw new SFException(ErrorCode.INTERNAL_ERROR, "Empty column fields");
     }
@@ -432,7 +445,7 @@ public InsertValidationResponse insertRows(
     try {
       this.channelState.updateInsertStats(System.currentTimeMillis(), this.bufferedRowCount);
       IngestionStrategy<T> ingestionStrategy = createIngestionStrategy(onErrorOption);
-      response = ingestionStrategy.insertRows(this, rows, offsetToken);
+      response = ingestionStrategy.insertRows(this, rows, startOffsetToken, endOffsetToken);
     } finally {
       this.tempStatsMap.values().forEach(RowBufferStats::reset);
       clearTempRows();
@@ -470,7 +483,7 @@ public ChannelData<T> flush(final String filePath) {
           oldRowCount = this.bufferedRowCount;
           oldBufferSize = this.bufferSize;
           oldRowSequencer = this.channelState.incrementAndGetRowSequencer();
-          oldOffsetToken = this.channelState.getOffsetToken();
+          oldOffsetToken = this.channelState.getEndOffsetToken();
           oldColumnEps = new HashMap<>(this.statsMap);
           oldMinMaxInsertTimeInMs =
               new Pair<>(
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelRuntimeState.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelRuntimeState.java
index 26314db3e..8fa7720d3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelRuntimeState.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelRuntimeState.java
@@ -11,8 +11,11 @@ class ChannelRuntimeState {
   // Indicates whether the channel is still valid
   private volatile boolean isValid;
 
-  // The channel's current offset token
-  private volatile String offsetToken;
+  // The channel's current start offset token
+  private volatile String startOffsetToken;
+
+  // The channel's current end offset token
+  private volatile String endOffsetToken;
 
   // The channel's current row sequencer
   private final AtomicLong rowSequencer;
@@ -21,8 +24,8 @@ class ChannelRuntimeState {
   private Long firstInsertInMs;
   private Long lastInsertInMs;
 
-  ChannelRuntimeState(String offsetToken, long rowSequencer, boolean isValid) {
-    this.offsetToken = offsetToken;
+  ChannelRuntimeState(String endOffsetToken, long rowSequencer, boolean isValid) {
+    this.endOffsetToken = endOffsetToken;
     this.rowSequencer = new AtomicLong(rowSequencer);
     this.isValid = isValid;
   }
@@ -41,9 +44,14 @@ void invalidate() {
     isValid = false;
   }
 
-  /** @return current offset token */
-  String getOffsetToken() {
-    return offsetToken;
+  /** @return current end offset token */
+  String getEndOffsetToken() {
+    return endOffsetToken;
+  }
+
+  /** @return current start offset token */
+  String getStartOffsetToken() {
+    return startOffsetToken;
   }
 
   /** @return current offset token after first incrementing it by one. */
@@ -57,12 +65,16 @@ long getRowSequencer() {
   }
 
   /**
-   * Updates the channel's offset token.
+   * Updates the channel's start and end offset token.
    *
-   * @param offsetToken new offset token
+   * @param startOffsetToken new start offset token of the batch
+   * @param endOffsetToken new end offset token
    */
-  void setOffsetToken(String offsetToken) {
-    this.offsetToken = offsetToken;
+  void updateOffsetToken(String startOffsetToken, String endOffsetToken, int rowCount) {
+    if (rowCount == 0) {
+      this.startOffsetToken = startOffsetToken;
+    }
+    this.endOffsetToken = endOffsetToken;
   }
 
   /** Update the insert stats for the current row buffer whenever needed */
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/IngestionStrategy.java b/src/main/java/net/snowflake/ingest/streaming/internal/IngestionStrategy.java
index defac31c4..2c44b50c4 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/IngestionStrategy.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/IngestionStrategy.java
@@ -13,9 +13,13 @@ public interface IngestionStrategy<T> {
    * Insert a batch of rows into the row buffer
    *
    * @param rows input row
-   * @param offsetToken offset token of the latest row in the batch
+   * @param startOffsetToken start offset token of the batch
+   * @param endOffsetToken offset token of the latest row in the batch
    * @return insert response that possibly contains errors because of insertion failures
    */
   InsertValidationResponse insertRows(
-      AbstractRowBuffer<T> rowBuffer, Iterable<Map<String, Object>> rows, String offsetToken);
+      AbstractRowBuffer<T> rowBuffer,
+      Iterable<Map<String, Object>> rows,
+      String startOffsetToken,
+      String endOffsetToken);
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index c587d52aa..eeae08932 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -48,7 +48,6 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
 
   private ByteArrayOutputStream fileOutput;
   private final List<List<Object>> tempData;
-  private final String channelName;
 
   private MessageType schema;
 
@@ -71,7 +70,6 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
     this.metadata = new HashMap<>();
     this.data = new ArrayList<>();
     this.tempData = new ArrayList<>();
-    this.channelName = fullyQualifiedChannelName;
   }
 
   @Override
@@ -121,7 +119,7 @@ private void createFileWriter() {
                 fileOutput,
                 schema,
                 metadata,
-                channelName,
+                channelFullyQualifiedName,
                 clientBufferParameters.getMaxChunkSizeInBytes(),
                 clientBufferParameters.getBdecParquetCompression());
       } else {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
index d7ba6dbd9..02905c02e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBuffer.java
@@ -26,10 +26,12 @@ interface RowBuffer<T> {
    * Insert a batch of rows into the row buffer
    *
    * @param rows input row
-   * @param offsetToken offset token of the latest row in the batch
+   * @param startOffsetToken start offset token of the batch
+   * @param endOffsetToken offset token of the latest row in the batch
    * @return insert response that possibly contains errors because of insertion failures
    */
-  InsertValidationResponse insertRows(Iterable<Map<String, Object>> rows, String offsetToken);
+  InsertValidationResponse insertRows(
+      Iterable<Map<String, Object>> rows, String startOffsetToken, String endOffsetToken);
 
   /**
    * Flush the data in the row buffer by taking the ownership of the old vectors and pass all the
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 756f9b5e8..695f5632b 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -19,6 +19,7 @@
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
+import javax.annotation.Nullable;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
@@ -102,7 +103,7 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
       String dbName,
       String schemaName,
       String tableName,
-      String offsetToken,
+      String endOffsetToken,
       Long channelSequencer,
       Long rowSequencer,
       SnowflakeStreamingIngestClientInternal<T> client,
@@ -116,7 +117,7 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
     this.channelFlushContext =
         new ChannelFlushContext(
             name, dbName, schemaName, tableName, channelSequencer, encryptionKey, encryptionKeyId);
-    this.channelState = new ChannelRuntimeState(offsetToken, rowSequencer, true);
+    this.channelState = new ChannelRuntimeState(endOffsetToken, rowSequencer, true);
     this.rowBuffer =
         AbstractRowBuffer.createRowBuffer(
             onErrorOption,
@@ -322,7 +323,7 @@ void setupSchema(List<ColumnMetadata> columns) {
    */
   @Override
   public InsertValidationResponse insertRow(Map<String, Object> row, String offsetToken) {
-    return insertRows(Collections.singletonList(row), offsetToken);
+    return insertRows(Collections.singletonList(row), offsetToken, offsetToken);
   }
 
   /**
@@ -332,16 +333,22 @@ public InsertValidationResponse insertRow(Map<String, Object> row, String offset
    */
 
   /**
-   * Each row is represented using Map where the key is column name and the value is a row of data
+   * Insert a batch of rows into the channel, each row is represented using Map where the key is
+   * column name and the value is a row of data. See {@link
+   * SnowflakeStreamingIngestChannel#insertRow(Map, String)} for more information about accepted
+   * values.
    *
    * @param rows object data to write
-   * @param offsetToken offset of last row in the row-set, used for replay in case of failures
+   * @param startOffsetToken start offset of the batch/row-set
+   * @param endOffsetToken end offset of the batch/row-set, used for replay in case of failures, *
+   *     It could be null if you don't plan on replaying or can't replay
    * @return insert response that possibly contains errors because of insertion failures
-   * @throws SFException when the channel is invalid or closed
    */
   @Override
   public InsertValidationResponse insertRows(
-      Iterable<Map<String, Object>> rows, String offsetToken) {
+      Iterable<Map<String, Object>> rows,
+      @Nullable String startOffsetToken,
+      @Nullable String endOffsetToken) {
     throttleInsertIfNeeded(new MemoryInfoProviderFromRuntime());
     checkValidation();
 
@@ -356,7 +363,8 @@ public InsertValidationResponse insertRows(
     final List<Map<String, Object>> rowsCopy = new LinkedList<>();
     rows.forEach(r -> rowsCopy.add(new LinkedHashMap<>(r)));
 
-    InsertValidationResponse response = this.rowBuffer.insertRows(rowsCopy, offsetToken);
+    InsertValidationResponse response =
+        this.rowBuffer.insertRows(rowsCopy, startOffsetToken, endOffsetToken);
 
     // Start flush task if the chunk size reaches a certain size
     // TODO: Checking table/chunk level size reduces throughput a lot, we may want to check it only
@@ -369,6 +377,16 @@ public InsertValidationResponse insertRows(
     return response;
   }
 
+  /**
+   * Insert a batch of rows into the channel with the end offset token only, please see {@link
+   * SnowflakeStreamingIngestChannel#insertRows(Iterable, String, String)} for more information.
+   */
+  @Override
+  public InsertValidationResponse insertRows(
+      Iterable<Map<String, Object>> rows, String offsetToken) {
+    return insertRows(rows, null, offsetToken);
+  }
+
   /** Collect the row size from row buffer if required */
   void collectRowSize(float rowSize) {
     if (this.owningClient.inputThroughput != null) {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 071f5f7c4..898642fd1 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -796,12 +796,14 @@ List<SnowflakeStreamingIngestChannelInternal<?>> verifyChannelsAreFullyCommitted
         long rowSequencer = channel.getChannelState().getRowSequencer();
         logger.logInfo(
             "Get channel status name={}, status={}, clientSequencer={}, rowSequencer={},"
-                + " offsetToken={}, persistedRowSequencer={}, persistedOffsetToken={}",
+                + " startOffsetToken={}, endOffsetToken={}, persistedRowSequencer={},"
+                + " persistedOffsetToken={}",
             channel.getName(),
             channelStatus.getStatusCode(),
             channel.getChannelSequencer(),
             rowSequencer,
-            channel.getChannelState().getOffsetToken(),
+            channel.getChannelState().getStartOffsetToken(),
+            channel.getChannelState().getEndOffsetToken(),
             channelStatus.getPersistedRowSequencer(),
             channelStatus.getPersistedOffsetToken());
         if (channelStatus.getStatusCode() != RESPONSE_SUCCESS) {
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
index a32b3a25d..86cece9c7 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/ParameterProviderTest.java
@@ -310,15 +310,15 @@ public void testValidCompressionAlgorithmsAndWithUppercaseLowerCase() {
         });
     List<String> zstdValues = Arrays.asList("ZSTD", "zstd", "Zstd", "zStd");
     zstdValues.forEach(
-            v -> {
-              Properties prop = new Properties();
-              Map<String, Object> parameterMap = getStartingParameterMap();
-              parameterMap.put(ParameterProvider.BDEC_PARQUET_COMPRESSION_ALGORITHM, v);
-              ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
-              Assert.assertEquals(
-                      Constants.BdecParquetCompression.ZSTD,
-                      parameterProvider.getBdecParquetCompressionAlgorithm());
-            });
+        v -> {
+          Properties prop = new Properties();
+          Map<String, Object> parameterMap = getStartingParameterMap();
+          parameterMap.put(ParameterProvider.BDEC_PARQUET_COMPRESSION_ALGORITHM, v);
+          ParameterProvider parameterProvider = new ParameterProvider(parameterMap, prop);
+          Assert.assertEquals(
+              Constants.BdecParquetCompression.ZSTD,
+              parameterProvider.getBdecParquetCompressionAlgorithm());
+        });
   }
 
   @Test
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index f26c999b7..178f1fb69 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -301,7 +301,7 @@ public void testRowIndexWithMultipleRowsWithErrorHelper(AbstractRowBuffer<?> row
     row.put("colChar", "1111111111111111111111"); // too big
     rows.add(row);
 
-    InsertValidationResponse response = rowBuffer.insertRows(rows, null);
+    InsertValidationResponse response = rowBuffer.insertRows(rows, null, null);
     Assert.assertTrue(response.hasErrors());
 
     Assert.assertEquals(2, response.getErrorRowCount());
@@ -355,7 +355,8 @@ private void testStringLengthHelper(AbstractRowBuffer<?> rowBuffer) {
     row.put("colDecimal", 1.23);
     row.put("colChar", "1234567890"); // still fits
 
-    InsertValidationResponse response = rowBuffer.insertRows(Collections.singletonList(row), null);
+    InsertValidationResponse response =
+        rowBuffer.insertRows(Collections.singletonList(row), null, null);
     Assert.assertFalse(response.hasErrors());
 
     row.put("colTinyInt", (byte) 1);
@@ -367,7 +368,7 @@ private void testStringLengthHelper(AbstractRowBuffer<?> rowBuffer) {
     row.put("colChar", "1111111111111111111111"); // too big
 
     if (rowBuffer.onErrorOption == OpenChannelRequest.OnErrorOption.CONTINUE) {
-      response = rowBuffer.insertRows(Collections.singletonList(row), null);
+      response = rowBuffer.insertRows(Collections.singletonList(row), null, null);
       Assert.assertTrue(response.hasErrors());
       Assert.assertEquals(1, response.getErrorRowCount());
       Assert.assertEquals(
@@ -376,7 +377,7 @@ private void testStringLengthHelper(AbstractRowBuffer<?> rowBuffer) {
       Assert.assertTrue(response.getInsertErrors().get(0).getMessage().contains("String too long"));
     } else {
       try {
-        rowBuffer.insertRows(Collections.singletonList(row), null);
+        rowBuffer.insertRows(Collections.singletonList(row), null, null);
       } catch (SFException e) {
         Assert.assertEquals(ErrorCode.INVALID_VALUE_ROW.getMessageCode(), e.getVendorCode());
       }
@@ -400,7 +401,8 @@ private void testInsertRowHelper(AbstractRowBuffer<?> rowBuffer) {
     row.put("colDecimal", 1.23);
     row.put("colChar", "2");
 
-    InsertValidationResponse response = rowBuffer.insertRows(Collections.singletonList(row), null);
+    InsertValidationResponse response =
+        rowBuffer.insertRows(Collections.singletonList(row), null, null);
     Assert.assertFalse(response.hasErrors());
   }
 
@@ -421,7 +423,8 @@ private void testInsertNullRowHelper(AbstractRowBuffer<?> rowBuffer) {
     row.put("colDecimal", null);
     row.put("colChar", null);
 
-    InsertValidationResponse response = rowBuffer.insertRows(Collections.singletonList(row), null);
+    InsertValidationResponse response =
+        rowBuffer.insertRows(Collections.singletonList(row), null, null);
     Assert.assertFalse(response.hasErrors());
   }
 
@@ -451,7 +454,7 @@ private void testInsertRowsHelper(AbstractRowBuffer<?> rowBuffer) {
     row2.put("colDecimal", 2.34);
     row2.put("colChar", "3");
 
-    InsertValidationResponse response = rowBuffer.insertRows(Arrays.asList(row1, row2), null);
+    InsertValidationResponse response = rowBuffer.insertRows(Arrays.asList(row1, row2), null, null);
     Assert.assertFalse(response.hasErrors());
   }
 
@@ -483,7 +486,7 @@ private void testFlushHelper(AbstractRowBuffer<?> rowBuffer) {
     row2.put("colChar", "3");
 
     InsertValidationResponse response =
-        rowBuffer.insertRows(Arrays.asList(row1, row2), offsetToken);
+        rowBuffer.insertRows(Arrays.asList(row1, row2), offsetToken, offsetToken);
     Assert.assertFalse(response.hasErrors());
     float bufferSize = rowBuffer.getSize();
 
@@ -525,7 +528,7 @@ private void testDoubleQuotesColumnNameHelper(OpenChannelRequest.OnErrorOption o
     row.put("\"colDoubleQuotes\"", 1);
 
     InsertValidationResponse response =
-        innerBuffer.insertRows(Collections.singletonList(row), null);
+        innerBuffer.insertRows(Collections.singletonList(row), null, null);
     Assert.assertFalse(response.hasErrors());
   }
 
@@ -646,7 +649,8 @@ private void testE2EHelper(AbstractRowBuffer<?> rowBuffer) {
     row1.put("colDecimal", 4);
     row1.put("colChar", "2");
 
-    InsertValidationResponse response = rowBuffer.insertRows(Collections.singletonList(row1), null);
+    InsertValidationResponse response =
+        rowBuffer.insertRows(Collections.singletonList(row1), null, null);
     Assert.assertFalse(response.hasErrors());
 
     Assert.assertEquals((byte) 10, rowBuffer.getVectorValueAt("colTinyInt", 0));
@@ -683,14 +687,14 @@ private void testE2ETimestampErrorsHelper(AbstractRowBuffer<?> innerBuffer) {
 
     if (innerBuffer.onErrorOption == OpenChannelRequest.OnErrorOption.CONTINUE) {
       InsertValidationResponse response =
-          innerBuffer.insertRows(Collections.singletonList(row), null);
+          innerBuffer.insertRows(Collections.singletonList(row), null, null);
       Assert.assertTrue(response.hasErrors());
       Assert.assertEquals(
           ErrorCode.INVALID_FORMAT_ROW.getMessageCode(),
           response.getInsertErrors().get(0).getException().getVendorCode());
     } else {
       try {
-        innerBuffer.insertRows(Collections.singletonList(row), null);
+        innerBuffer.insertRows(Collections.singletonList(row), null, null);
       } catch (SFException e) {
         Assert.assertEquals(ErrorCode.INVALID_FORMAT_ROW.getMessageCode(), e.getVendorCode());
       }
@@ -724,7 +728,7 @@ private void testStatsE2EHelper(AbstractRowBuffer<?> rowBuffer) {
     row2.put("colChar", "alice");
 
     final String filename = "testStatsE2EHelper_streaming.bdec";
-    InsertValidationResponse response = rowBuffer.insertRows(Arrays.asList(row1, row2), null);
+    InsertValidationResponse response = rowBuffer.insertRows(Arrays.asList(row1, row2), null, null);
     Assert.assertFalse(response.hasErrors());
     ChannelData<?> result = rowBuffer.flush(filename);
     Map<String, RowBufferStats> columnEpStats = result.getColumnEps();
@@ -831,7 +835,7 @@ private void testStatsE2ETimestampHelper(OpenChannelRequest.OnErrorOption onErro
     row3.put("COLTIMESTAMPLTZ_SB16_SCALE6", null);
 
     InsertValidationResponse response =
-        innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null);
+        innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null, null);
     Assert.assertFalse(response.hasErrors());
     ChannelData<?> result = innerBuffer.flush("my_snowpipe_streaming.bdec");
     Assert.assertEquals(3, result.getRowCount());
@@ -893,7 +897,7 @@ private void testE2EDateHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
     row3.put("COLDATE", null);
 
     InsertValidationResponse response =
-        innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null);
+        innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null, null);
     Assert.assertFalse(response.hasErrors());
 
     // Check data was inserted into the buffer correctly
@@ -954,7 +958,7 @@ private void testE2ETimeHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
     row3.put("COLTIMESB8", null);
 
     InsertValidationResponse response =
-        innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null);
+        innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null, null);
     Assert.assertFalse(response.hasErrors());
 
     // Check data was inserted into the buffer correctly
@@ -1013,13 +1017,13 @@ private void testMaxInsertRowsBatchSizeHelper(OpenChannelRequest.OnErrorOption o
     }
 
     // Insert rows should succeed
-    innerBuffer.insertRows(rows, "");
+    innerBuffer.insertRows(rows, "", "");
 
     // After adding another row, it should fail due to too large batch of rows passed to
     // insertRows() in one go
     rows.add(Collections.singletonMap("COLBINARY", arr));
     try {
-      innerBuffer.insertRows(rows, "");
+      innerBuffer.insertRows(rows, "", "");
       Assert.fail("Inserting rows should have failed");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.MAX_BATCH_SIZE_EXCEEDED.getMessageCode(), e.getVendorCode());
@@ -1048,19 +1052,20 @@ private void testNullableCheckHelper(OpenChannelRequest.OnErrorOption onErrorOpt
     Map<String, Object> row = new HashMap<>();
     row.put("COLBOOLEAN", true);
 
-    InsertValidationResponse response = innerBuffer.insertRows(Collections.singletonList(row), "1");
+    InsertValidationResponse response =
+        innerBuffer.insertRows(Collections.singletonList(row), "1", "1");
     Assert.assertFalse(response.hasErrors());
 
     row.put("COLBOOLEAN", null);
     if (innerBuffer.onErrorOption == OpenChannelRequest.OnErrorOption.CONTINUE) {
-      response = innerBuffer.insertRows(Collections.singletonList(row), "1");
+      response = innerBuffer.insertRows(Collections.singletonList(row), "1", "1");
       Assert.assertTrue(response.hasErrors());
       Assert.assertEquals(
           ErrorCode.INVALID_FORMAT_ROW.getMessageCode(),
           response.getInsertErrors().get(0).getException().getVendorCode());
     } else {
       try {
-        innerBuffer.insertRows(Collections.singletonList(row), "1");
+        innerBuffer.insertRows(Collections.singletonList(row), "1", "1");
       } catch (SFException e) {
         Assert.assertEquals(ErrorCode.INVALID_FORMAT_ROW.getMessageCode(), e.getVendorCode());
       }
@@ -1097,13 +1102,14 @@ private void testMissingColumnCheckHelper(OpenChannelRequest.OnErrorOption onErr
     Map<String, Object> row = new HashMap<>();
     row.put("COLBOOLEAN", true);
 
-    InsertValidationResponse response = innerBuffer.insertRows(Collections.singletonList(row), "1");
+    InsertValidationResponse response =
+        innerBuffer.insertRows(Collections.singletonList(row), "1", "1");
     Assert.assertFalse(response.hasErrors());
 
     Map<String, Object> row2 = new HashMap<>();
     row2.put("COLBOOLEAN2", true);
     if (innerBuffer.onErrorOption == OpenChannelRequest.OnErrorOption.CONTINUE) {
-      response = innerBuffer.insertRows(Collections.singletonList(row2), "2");
+      response = innerBuffer.insertRows(Collections.singletonList(row2), "1", "2");
       Assert.assertTrue(response.hasErrors());
       InsertValidationResponse.InsertError error = response.getInsertErrors().get(0);
       Assert.assertEquals(
@@ -1112,7 +1118,7 @@ private void testMissingColumnCheckHelper(OpenChannelRequest.OnErrorOption onErr
           Collections.singletonList("COLBOOLEAN"), error.getMissingNotNullColNames());
     } else {
       try {
-        innerBuffer.insertRows(Collections.singletonList(row2), "2");
+        innerBuffer.insertRows(Collections.singletonList(row2), "1", "2");
       } catch (SFException e) {
         Assert.assertEquals(ErrorCode.INVALID_FORMAT_ROW.getMessageCode(), e.getVendorCode());
       }
@@ -1137,7 +1143,8 @@ public void testExtraColumnsCheck() {
     row.put("COLBOOLEAN2", true);
     row.put("COLBOOLEAN3", true);
 
-    InsertValidationResponse response = innerBuffer.insertRows(Collections.singletonList(row), "1");
+    InsertValidationResponse response =
+        innerBuffer.insertRows(Collections.singletonList(row), "1", "1");
     Assert.assertTrue(response.hasErrors());
     InsertValidationResponse.InsertError error = response.getInsertErrors().get(0);
     Assert.assertEquals(
@@ -1199,7 +1206,7 @@ private void doTestFailureHalfwayThroughColumnProcessing(
 
     for (Map<String, Object> row : Arrays.asList(row1, row2, row3)) {
       try {
-        innerBuffer.insertRows(Collections.singletonList(row), "");
+        innerBuffer.insertRows(Collections.singletonList(row), "", "");
       } catch (Exception ignored) {
         // we ignore exceptions, for ABORT option there will be some, but we don't care in this test
       }
@@ -1256,7 +1263,7 @@ private void testE2EBooleanHelper(OpenChannelRequest.OnErrorOption onErrorOption
 
     // innerBuffer.insertRows(Collections.singletonList(row1));
     InsertValidationResponse response =
-        innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null);
+        innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null, null);
     Assert.assertFalse(response.hasErrors());
 
     // Check data was inserted into the buffer correctly
@@ -1307,7 +1314,7 @@ private void testE2EBinaryHelper(OpenChannelRequest.OnErrorOption onErrorOption)
     row3.put("COLBINARY", null);
 
     InsertValidationResponse response =
-        innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null);
+        innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null, null);
     Assert.assertFalse(response.hasErrors());
 
     // Check data was inserted into the buffer correctly
@@ -1363,7 +1370,7 @@ private void testE2ERealHelper(OpenChannelRequest.OnErrorOption onErrorOption) {
     row3.put("COLREAL", null);
 
     InsertValidationResponse response =
-        innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null);
+        innerBuffer.insertRows(Arrays.asList(row1, row2, row3), null, null);
     Assert.assertFalse(response.hasErrors());
 
     // Check data was inserted into the buffer correctly
@@ -1399,7 +1406,8 @@ public void testOnErrorAbortFailures() {
     Map<String, Object> row = new HashMap<>();
     row.put("COLDECIMAL", 1);
 
-    InsertValidationResponse response = innerBuffer.insertRows(Collections.singletonList(row), "1");
+    InsertValidationResponse response =
+        innerBuffer.insertRows(Collections.singletonList(row), "1", "1");
     Assert.assertFalse(response.hasErrors());
 
     Assert.assertEquals(1, innerBuffer.bufferedRowCount);
@@ -1413,7 +1421,7 @@ public void testOnErrorAbortFailures() {
 
     Map<String, Object> row2 = new HashMap<>();
     row2.put("COLDECIMAL", 2);
-    response = innerBuffer.insertRows(Collections.singletonList(row2), "2");
+    response = innerBuffer.insertRows(Collections.singletonList(row2), "1", "2");
     Assert.assertFalse(response.hasErrors());
 
     Assert.assertEquals(2, innerBuffer.bufferedRowCount);
@@ -1428,7 +1436,7 @@ public void testOnErrorAbortFailures() {
     Map<String, Object> row3 = new HashMap<>();
     row3.put("COLDECIMAL", true);
     try {
-      innerBuffer.insertRows(Collections.singletonList(row3), "3");
+      innerBuffer.insertRows(Collections.singletonList(row3), "1", "3");
     } catch (SFException e) {
       Assert.assertEquals(ErrorCode.INVALID_FORMAT_ROW.getMessageCode(), e.getVendorCode());
     }
@@ -1443,7 +1451,7 @@ public void testOnErrorAbortFailures() {
     Assert.assertNull(innerBuffer.tempStatsMap.get("COLDECIMAL").getCurrentMinIntValue());
 
     row3.put("COLDECIMAL", 3);
-    response = innerBuffer.insertRows(Collections.singletonList(row3), "3");
+    response = innerBuffer.insertRows(Collections.singletonList(row3), "1", "3");
     Assert.assertFalse(response.hasErrors());
     Assert.assertEquals(3, innerBuffer.bufferedRowCount);
     Assert.assertEquals(0, innerBuffer.getTempRowCount());
@@ -1477,7 +1485,8 @@ public void testOnErrorAbortSkipBatch() {
     Map<String, Object> row = new HashMap<>();
     row.put("COLDECIMAL", 1);
 
-    InsertValidationResponse response = innerBuffer.insertRows(Collections.singletonList(row), "1");
+    InsertValidationResponse response =
+        innerBuffer.insertRows(Collections.singletonList(row), "1", "1");
     Assert.assertFalse(response.hasErrors());
 
     Assert.assertEquals(1, innerBuffer.bufferedRowCount);
@@ -1494,7 +1503,7 @@ public void testOnErrorAbortSkipBatch() {
     Map<String, Object> row3 = new HashMap<>();
     row3.put("COLDECIMAL", true);
 
-    response = innerBuffer.insertRows(Arrays.asList(row2, row3), "3");
+    response = innerBuffer.insertRows(Arrays.asList(row2, row3), "1", "3");
     Assert.assertTrue(response.hasErrors());
 
     Assert.assertEquals(1, innerBuffer.bufferedRowCount);
@@ -1516,7 +1525,7 @@ public void testOnErrorAbortSkipBatch() {
     Assert.assertNull(innerBuffer.tempStatsMap.get("COLDECIMAL").getCurrentMinIntValue());
 
     row3.put("COLDECIMAL", 3);
-    response = innerBuffer.insertRows(Arrays.asList(row2, row3), "3");
+    response = innerBuffer.insertRows(Arrays.asList(row2, row3), "1", "3");
     Assert.assertFalse(response.hasErrors());
     Assert.assertEquals(3, innerBuffer.bufferedRowCount);
     Assert.assertEquals(0, innerBuffer.getTempRowCount());
@@ -1567,7 +1576,7 @@ private void testE2EVariantHelper(OpenChannelRequest.OnErrorOption onErrorOption
     row5.put("COLVARIANT", 3);
 
     InsertValidationResponse response =
-        innerBuffer.insertRows(Arrays.asList(row1, row2, row3, row4, row5), null);
+        innerBuffer.insertRows(Arrays.asList(row1, row2, row3, row4, row5), null, null);
     Assert.assertFalse(response.hasErrors());
 
     // Check data was inserted into the buffer correctly
@@ -1605,7 +1614,7 @@ private void testE2EObjectHelper(OpenChannelRequest.OnErrorOption onErrorOption)
     Map<String, Object> row1 = new HashMap<>();
     row1.put("COLOBJECT", "{\"key\":1}");
 
-    InsertValidationResponse response = innerBuffer.insertRows(Arrays.asList(row1), null);
+    InsertValidationResponse response = innerBuffer.insertRows(Arrays.asList(row1), null, null);
     Assert.assertFalse(response.hasErrors());
 
     // Check data was inserted into the buffer correctly
@@ -1651,7 +1660,7 @@ private void testE2EArrayHelper(OpenChannelRequest.OnErrorOption onErrorOption)
     row5.put("COLARRAY", Arrays.asList(1, 2, 3));
 
     InsertValidationResponse response =
-        innerBuffer.insertRows(Arrays.asList(row1, row2, row3, row4, row5), null);
+        innerBuffer.insertRows(Arrays.asList(row1, row2, row3, row4, row5), null, null);
     Assert.assertFalse(response.hasErrors());
 
     // Check data was inserted into the buffer correctly
@@ -1692,9 +1701,9 @@ public void testOnErrorAbortRowsWithError() {
     List<Map<String, Object>> validRows = new ArrayList<>();
     validRows.add(Collections.singletonMap("colChar", "a"));
 
-    InsertValidationResponse response = innerBufferOnErrorContinue.insertRows(validRows, "1");
+    InsertValidationResponse response = innerBufferOnErrorContinue.insertRows(validRows, "1", "1");
     Assert.assertFalse(response.hasErrors());
-    response = innerBufferOnErrorAbort.insertRows(validRows, "1");
+    response = innerBufferOnErrorAbort.insertRows(validRows, "1", "1");
     Assert.assertFalse(response.hasErrors());
 
     // insert one valid and one invalid row
@@ -1702,11 +1711,11 @@ public void testOnErrorAbortRowsWithError() {
     mixedRows.add(Collections.singletonMap("colChar", "b"));
     mixedRows.add(Collections.singletonMap("colChar", "1111111111111111111111")); // too big
 
-    response = innerBufferOnErrorContinue.insertRows(mixedRows, "3");
+    response = innerBufferOnErrorContinue.insertRows(mixedRows, "1", "3");
     Assert.assertTrue(response.hasErrors());
 
     Assert.assertThrows(
-        SFException.class, () -> innerBufferOnErrorAbort.insertRows(mixedRows, "3"));
+        SFException.class, () -> innerBufferOnErrorAbort.insertRows(mixedRows, "1", "3"));
 
     List<List<Object>> snapshotContinueParquet =
         ((ParquetChunkData) innerBufferOnErrorContinue.getSnapshot("fake/filePath").get()).rows;
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index d419fffc0..8e597ae64 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -143,7 +143,7 @@ public void testChannelFactorySuccess() {
     Assert.assertEquals(dbName, channel.getDBName());
     Assert.assertEquals(schemaName, channel.getSchemaName());
     Assert.assertEquals(tableName, channel.getTableName());
-    Assert.assertEquals(offsetToken, channel.getChannelState().getOffsetToken());
+    Assert.assertEquals(offsetToken, channel.getChannelState().getEndOffsetToken());
     Assert.assertEquals(channelSequencer, channel.getChannelSequencer());
     Assert.assertEquals(rowSequencer + 1L, channel.getChannelState().incrementAndGetRowSequencer());
     Assert.assertEquals(
@@ -576,14 +576,16 @@ public void testInsertRow() {
     Assert.assertFalse(response.hasErrors());
     response = channel.insertRows(Collections.singletonList(row), "2");
     Assert.assertFalse(response.hasErrors());
+    response = channel.insertRows(Collections.singletonList(row), "3", "3");
+    Assert.assertFalse(response.hasErrors());
     long insertEndTimeInMs = System.currentTimeMillis();
 
     // Get data again to verify the row is inserted
     data = channel.getData("my_snowpipe_streaming.bdec");
-    Assert.assertEquals(2, data.getRowCount());
+    Assert.assertEquals(3, data.getRowCount());
     Assert.assertEquals((Long) 1L, data.getRowSequencer());
     Assert.assertEquals(1, ((ChannelData<ParquetChunkData>) data).getVectors().rows.get(0).size());
-    Assert.assertEquals("2", data.getOffsetToken());
+    Assert.assertEquals("3", data.getOffsetToken());
     Assert.assertTrue(data.getBufferSize() > 0);
     Assert.assertTrue(insertStartTimeInMs <= data.getMinMaxInsertTimeInMs().getFirst());
     Assert.assertTrue(insertEndTimeInMs >= data.getMinMaxInsertTimeInMs().getSecond());
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index 0eece5f02..1107aad89 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -460,7 +460,7 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
         ChannelMetadata.builder()
             .setOwningChannelFromContext(channel.getChannelContext())
             .setRowSequencer(channel.getChannelState().incrementAndGetRowSequencer())
-            .setOffsetToken(channel.getChannelState().getOffsetToken())
+            .setOffsetToken(channel.getChannelState().getEndOffsetToken())
             .build();
 
     Map<String, RowBufferStats> columnEps = new HashMap<>();
@@ -514,25 +514,25 @@ private Pair<List<BlobMetadata>, Set<ChunkRegisterStatus>> getRetryBlobMetadata(
         ChannelMetadata.builder()
             .setOwningChannelFromContext(channel1.getChannelContext())
             .setRowSequencer(channel1.getChannelState().incrementAndGetRowSequencer())
-            .setOffsetToken(channel1.getChannelState().getOffsetToken())
+            .setOffsetToken(channel1.getChannelState().getEndOffsetToken())
             .build();
     ChannelMetadata channelMetadata2 =
         ChannelMetadata.builder()
             .setOwningChannelFromContext(channel2.getChannelContext())
             .setRowSequencer(channel2.getChannelState().incrementAndGetRowSequencer())
-            .setOffsetToken(channel2.getChannelState().getOffsetToken())
+            .setOffsetToken(channel2.getChannelState().getEndOffsetToken())
             .build();
     ChannelMetadata channelMetadata3 =
         ChannelMetadata.builder()
             .setOwningChannelFromContext(channel3.getChannelContext())
             .setRowSequencer(channel3.getChannelState().incrementAndGetRowSequencer())
-            .setOffsetToken(channel3.getChannelState().getOffsetToken())
+            .setOffsetToken(channel3.getChannelState().getEndOffsetToken())
             .build();
     ChannelMetadata channelMetadata4 =
         ChannelMetadata.builder()
             .setOwningChannelFromContext(channel4.getChannelContext())
             .setRowSequencer(channel4.getChannelState().incrementAndGetRowSequencer())
-            .setOffsetToken(channel4.getChannelState().getOffsetToken())
+            .setOffsetToken(channel4.getChannelState().getEndOffsetToken())
             .build();
 
     List<BlobMetadata> blobs = new ArrayList<>();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
index 62494037c..ead26acd6 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestBigFilesIT.java
@@ -40,8 +40,7 @@ public static Object[] compressionAlgorithms() {
     return new Object[] {"GZIP", "ZSTD"};
   }
 
-  @Parameter
-  public String compressionAlgorithm;
+  @Parameter public String compressionAlgorithm;
 
   @Before
   public void beforeAll() throws Exception {
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 38769e0b1..f3ee8dfe6 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -75,8 +75,7 @@ public static Object[] compressionAlgorithms() {
     return new Object[] {"GZIP", "ZSTD"};
   }
 
-  @Parameter
-  public String compressionAlgorithm;
+  @Parameter public String compressionAlgorithm;
 
   @Before
   public void beforeAll() throws Exception {
@@ -963,6 +962,116 @@ public void testAbortOnErrorSkipBatch() throws Exception {
     Assert.fail("Row sequencer not updated before timeout");
   }
 
+  @Test
+  public void testInsertRowsWithValidStartOffsetToken() throws Exception {
+    String insertRowsWithValidStartOffsetToken = "insert_rows_with_valid_start_offset_token";
+    jdbcConnection
+        .createStatement()
+        .execute(
+            String.format(
+                "create or replace table %s (c1 int);", insertRowsWithValidStartOffsetToken));
+
+    OpenChannelRequest request =
+        OpenChannelRequest.builder("CHANNEL")
+            .setDBName(testDb)
+            .setSchemaName(TEST_SCHEMA)
+            .setTableName(insertRowsWithValidStartOffsetToken)
+            .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+            .build();
+
+    // Open a streaming ingest channel from the given client
+    SnowflakeStreamingIngestChannel channel = client.openChannel(request);
+    Map<String, Object> row1 = new HashMap<>();
+    row1.put("c1", 1);
+    Map<String, Object> row2 = new HashMap<>();
+    row2.put("c1", 2);
+    Map<String, Object> row3 = new HashMap<>();
+    row3.put("c1", "a");
+
+    verifyInsertValidationResponse(channel.insertRow(row1, "1"));
+
+    InsertValidationResponse response =
+        channel.insertRows(Arrays.asList(row1, row2, row3), "2", "4");
+    Assert.assertTrue(response.hasErrors());
+
+    // Close the channel after insertion
+    channel.close().get();
+
+    for (int i = 1; i < 15; i++) {
+      if (channel.getLatestCommittedOffsetToken() != null
+          && channel.getLatestCommittedOffsetToken().equals("4")) {
+        ResultSet result =
+            jdbcConnection
+                .createStatement()
+                .executeQuery(
+                    String.format(
+                        "select count(c1), min(c1), max(c1) from %s.%s.%s",
+                        testDb, TEST_SCHEMA, insertRowsWithValidStartOffsetToken));
+        result.next();
+        Assert.assertEquals("3", result.getString(1));
+        Assert.assertEquals("1", result.getString(2));
+        Assert.assertEquals("2", result.getString(3));
+        return;
+      }
+      Thread.sleep(1000);
+    }
+    Assert.fail("Row sequencer not updated before timeout");
+  }
+
+  @Test
+  public void testInsertRowsWithInvalidStartOffsetToken() throws Exception {
+    String insertRowsWithInvalidStartOffsetToken = "insert_rows_with_invalid_start_offset_token";
+    jdbcConnection
+        .createStatement()
+        .execute(
+            String.format(
+                "create or replace table %s (c1 int);", insertRowsWithInvalidStartOffsetToken));
+
+    OpenChannelRequest request =
+        OpenChannelRequest.builder("CHANNEL")
+            .setDBName(testDb)
+            .setSchemaName(TEST_SCHEMA)
+            .setTableName(insertRowsWithInvalidStartOffsetToken)
+            .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+            .build();
+
+    // Open a streaming ingest channel from the given client
+    SnowflakeStreamingIngestChannel channel = client.openChannel(request);
+    Map<String, Object> row1 = new HashMap<>();
+    row1.put("c1", 1);
+    Map<String, Object> row2 = new HashMap<>();
+    row2.put("c1", 2);
+
+    // end-start != rowcount
+    channel.insertRows(Arrays.asList(row1, row2), "1", "4");
+
+    // offset != prevEnd+1
+    channel.insertRow(row1, "7");
+
+    // Close the channel after insertion
+    channel.close().get();
+
+    for (int i = 1; i < 15; i++) {
+      if (channel.getLatestCommittedOffsetToken() != null
+          && channel.getLatestCommittedOffsetToken().equals("7")) {
+        ResultSet result =
+            jdbcConnection
+                .createStatement()
+                .executeQuery(
+                    String.format(
+                        "select count(c1), min(c1), max(c1) from %s.%s.%s",
+                        testDb, TEST_SCHEMA, insertRowsWithInvalidStartOffsetToken));
+        result.next();
+        Assert.assertEquals("3", result.getString(1));
+        Assert.assertEquals("1", result.getString(2));
+        Assert.assertEquals("2", result.getString(3));
+        return;
+      }
+      Thread.sleep(1000);
+    }
+    Assert.fail("Row sequencer not updated before timeout");
+  }
+
   @Test
   public void testChannelClose() throws Exception {
     OpenChannelRequest request1 =
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index 6718a0aeb..4d510acf6 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -1,6 +1,7 @@
 package net.snowflake.ingest.streaming.internal.datatypes;
 
 import static net.snowflake.ingest.utils.Constants.ROLE;
+import static net.snowflake.ingest.utils.ParameterProvider.BDEC_PARQUET_COMPRESSION_ALGORITHM;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.math.BigDecimal;
@@ -22,7 +23,6 @@
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
 import net.snowflake.ingest.utils.Constants;
-import static net.snowflake.ingest.utils.ParameterProvider.BDEC_PARQUET_COMPRESSION_ALGORITHM;
 import net.snowflake.ingest.utils.SFException;
 import org.junit.After;
 import org.junit.Assert;
@@ -67,8 +67,7 @@ public static Object[] compressionAlgorithms() {
     return new Object[] {"GZIP", "ZSTD"};
   }
 
-  @Parameter
-  public String compressionAlgorithm;
+  @Parameter public String compressionAlgorithm;
 
   @Before
   public void before() throws Exception {

From f0be0f5024fc76bdbf501b5433827b75fde63913 Mon Sep 17 00:00:00 2001
From: Tyler Jones <59716821+sfc-gh-tjones@users.noreply.github.com>
Date: Tue, 20 Feb 2024 08:42:04 -0800
Subject: [PATCH 322/356] SNOW-967235 Sends start and end offsets from the
 Client SDK (#675)

This sends the start and end offset tokens for a channel when sending a
blob to Snowflake. Right now only the end offset is sent which makes
reasoning about ranges difficult.

@test verified via existing tests in `RowBufferTest` and
`SnowflakeStreamingIngestChannelTest`
---
 .../streaming/internal/AbstractRowBuffer.java |  3 +++
 .../streaming/internal/ChannelData.java       | 11 ++++++++++
 .../streaming/internal/ChannelMetadata.java   | 21 +++++++++++++++++++
 .../streaming/internal/ParquetFlusher.java    |  2 ++
 .../streaming/internal/RowBufferTest.java     |  1 +
 .../SnowflakeStreamingIngestChannelTest.java  |  1 +
 6 files changed, 39 insertions(+)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index e111266e9..78a8640df 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -470,6 +470,7 @@ public ChannelData<T> flush(final String filePath) {
       float oldBufferSize = 0F;
       long oldRowSequencer = 0;
       String oldOffsetToken = null;
+      String oldStartOffsetToken = null;
       Map<String, RowBufferStats> oldColumnEps = null;
       Pair<Long, Long> oldMinMaxInsertTimeInMs = null;
 
@@ -484,6 +485,7 @@ public ChannelData<T> flush(final String filePath) {
           oldBufferSize = this.bufferSize;
           oldRowSequencer = this.channelState.incrementAndGetRowSequencer();
           oldOffsetToken = this.channelState.getEndOffsetToken();
+          oldStartOffsetToken = this.channelState.getStartOffsetToken();
           oldColumnEps = new HashMap<>(this.statsMap);
           oldMinMaxInsertTimeInMs =
               new Pair<>(
@@ -508,6 +510,7 @@ public ChannelData<T> flush(final String filePath) {
         data.setBufferSize(oldBufferSize);
         data.setRowSequencer(oldRowSequencer);
         data.setOffsetToken(oldOffsetToken);
+        data.setStartOffsetToken(oldStartOffsetToken);
         data.setColumnEps(oldColumnEps);
         data.setMinMaxInsertTimeInMs(oldMinMaxInsertTimeInMs);
         data.setFlusherFactory(this::createFlusher);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
index cd4dabaa6..39260a940 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
@@ -20,6 +20,7 @@
 class ChannelData<T> {
   private Long rowSequencer;
   private String offsetToken;
+  private String startOffsetToken;
   private T vectors;
   private float bufferSize;
   private int rowCount;
@@ -98,10 +99,18 @@ String getOffsetToken() {
     return this.offsetToken;
   }
 
+  String getStartOffsetToken() {
+    return this.startOffsetToken;
+  }
+
   void setOffsetToken(String offsetToken) {
     this.offsetToken = offsetToken;
   }
 
+  void setStartOffsetToken(String startOffsetToken) {
+    this.startOffsetToken = startOffsetToken;
+  }
+
   T getVectors() {
     return this.vectors;
   }
@@ -157,6 +166,8 @@ public String toString() {
         + rowSequencer
         + ", offsetToken='"
         + offsetToken
+        + ", startOffsetToken='"
+        + startOffsetToken
         + '\''
         + ", bufferSize="
         + bufferSize
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelMetadata.java
index 6edb29e62..d9043f8d7 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelMetadata.java
@@ -17,6 +17,7 @@ class ChannelMetadata {
   private final Long clientSequencer;
   private final Long rowSequencer;
   @Nullable private final String offsetToken;
+  @Nullable private final String startOffsetToken;
 
   static Builder builder() {
     return new Builder();
@@ -28,6 +29,7 @@ static class Builder {
     private Long clientSequencer;
     private Long rowSequencer;
     @Nullable private String offsetToken; // offset token could be null
+    @Nullable private String startOffsetToken; // start offset token could be null
 
     Builder setOwningChannelFromContext(ChannelFlushContext channelFlushContext) {
       this.channelName = channelFlushContext.getName();
@@ -45,6 +47,12 @@ Builder setOffsetToken(String offsetToken) {
       return this;
     }
 
+
+    Builder setStartOffsetToken(String startOffsetToken) {
+      this.startOffsetToken = startOffsetToken;
+      return this;
+    }
+
     ChannelMetadata build() {
       return new ChannelMetadata(this);
     }
@@ -59,6 +67,7 @@ private ChannelMetadata(Builder builder) {
     this.clientSequencer = builder.clientSequencer;
     this.rowSequencer = builder.rowSequencer;
     this.offsetToken = builder.offsetToken;
+    this.startOffsetToken = builder.startOffsetToken;
   }
 
   @JsonProperty("channel_name")
@@ -81,4 +90,16 @@ Long getRowSequencer() {
   String getOffsetToken() {
     return this.offsetToken;
   }
+
+  @Nullable
+  @JsonProperty("start_offset_token")
+  String getStartOffsetToken() {
+    return this.startOffsetToken;
+  }
+
+  @Nullable
+  @JsonProperty("end_offset_token")
+  String getEndOffsetToken() {
+    return this.offsetToken;
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
index e81dbc0eb..94348fffc 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
@@ -74,6 +74,7 @@ private SerializationResult serializeFromParquetWriteBuffers(
               .setOwningChannelFromContext(data.getChannelContext())
               .setRowSequencer(data.getRowSequencer())
               .setOffsetToken(data.getOffsetToken())
+              .setStartOffsetToken(data.getStartOffsetToken())
               .build();
       // Add channel metadata to the metadata list
       channelsMetadataList.add(channelMetadata);
@@ -153,6 +154,7 @@ private SerializationResult serializeFromJavaObjects(
               .setOwningChannelFromContext(data.getChannelContext())
               .setRowSequencer(data.getRowSequencer())
               .setOffsetToken(data.getOffsetToken())
+              .setStartOffsetToken(data.getStartOffsetToken())
               .build();
       // Add channel metadata to the metadata list
       channelsMetadataList.add(channelMetadata);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 178f1fb69..523cacaee 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -495,6 +495,7 @@ private void testFlushHelper(AbstractRowBuffer<?> rowBuffer) {
     Assert.assertEquals(2, data.getRowCount());
     Assert.assertEquals((Long) 1L, data.getRowSequencer());
     Assert.assertEquals(offsetToken, data.getOffsetToken());
+    Assert.assertEquals(offsetToken, data.getStartOffsetToken());
     Assert.assertEquals(bufferSize, data.getBufferSize(), 0);
 
     final ParquetChunkData chunkData = (ParquetChunkData) data.getVectors();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 8e597ae64..6d1697cff 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -586,6 +586,7 @@ public void testInsertRow() {
     Assert.assertEquals((Long) 1L, data.getRowSequencer());
     Assert.assertEquals(1, ((ChannelData<ParquetChunkData>) data).getVectors().rows.get(0).size());
     Assert.assertEquals("3", data.getOffsetToken());
+    Assert.assertEquals("1", data.getStartOffsetToken());
     Assert.assertTrue(data.getBufferSize() > 0);
     Assert.assertTrue(insertStartTimeInMs <= data.getMinMaxInsertTimeInMs().getFirst());
     Assert.assertTrue(insertEndTimeInMs >= data.getMinMaxInsertTimeInMs().getSecond());

From 453743e62595a052d2a56c82a3b1b97c4a3e332b Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Tue, 20 Feb 2024 17:20:39 -0800
Subject: [PATCH 323/356] NO-SNOW: Two minor changes that address PR comments
 and fix code format (#679)

- Address the comment in 0c85b60 and add a test
- Fix some format issues added by a69e6f4
---
 .../streaming/internal/AbstractRowBuffer.java      |  6 +++---
 .../ingest/streaming/internal/ChannelData.java     | 14 +++++++-------
 .../ingest/streaming/internal/ChannelMetadata.java |  9 ++++-----
 .../streaming/internal/FileColumnProperties.java   |  2 +-
 .../ingest/streaming/internal/ParquetFlusher.java  |  4 ++--
 .../streaming/internal/ParquetRowBuffer.java       |  6 ++++--
 .../ingest/streaming/internal/RowBufferStats.java  |  8 +++++---
 .../streaming/internal/FlushServiceTest.java       |  4 ++--
 .../ingest/streaming/internal/RowBufferTest.java   | 13 +++++++------
 .../SnowflakeStreamingIngestChannelTest.java       |  2 +-
 10 files changed, 36 insertions(+), 32 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 78a8640df..6f7edd921 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -469,7 +469,7 @@ public ChannelData<T> flush(final String filePath) {
       int oldRowCount = 0;
       float oldBufferSize = 0F;
       long oldRowSequencer = 0;
-      String oldOffsetToken = null;
+      String oldEndOffsetToken = null;
       String oldStartOffsetToken = null;
       Map<String, RowBufferStats> oldColumnEps = null;
       Pair<Long, Long> oldMinMaxInsertTimeInMs = null;
@@ -484,7 +484,7 @@ public ChannelData<T> flush(final String filePath) {
           oldRowCount = this.bufferedRowCount;
           oldBufferSize = this.bufferSize;
           oldRowSequencer = this.channelState.incrementAndGetRowSequencer();
-          oldOffsetToken = this.channelState.getEndOffsetToken();
+          oldEndOffsetToken = this.channelState.getEndOffsetToken();
           oldStartOffsetToken = this.channelState.getStartOffsetToken();
           oldColumnEps = new HashMap<>(this.statsMap);
           oldMinMaxInsertTimeInMs =
@@ -509,7 +509,7 @@ public ChannelData<T> flush(final String filePath) {
         data.setRowCount(oldRowCount);
         data.setBufferSize(oldBufferSize);
         data.setRowSequencer(oldRowSequencer);
-        data.setOffsetToken(oldOffsetToken);
+        data.setEndOffsetToken(oldEndOffsetToken);
         data.setStartOffsetToken(oldStartOffsetToken);
         data.setColumnEps(oldColumnEps);
         data.setMinMaxInsertTimeInMs(oldMinMaxInsertTimeInMs);
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
index 39260a940..81f49d2fd 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelData.java
@@ -19,7 +19,7 @@
  */
 class ChannelData<T> {
   private Long rowSequencer;
-  private String offsetToken;
+  private String endOffsetToken;
   private String startOffsetToken;
   private T vectors;
   private float bufferSize;
@@ -95,16 +95,16 @@ void setRowSequencer(Long rowSequencer) {
     this.rowSequencer = rowSequencer;
   }
 
-  String getOffsetToken() {
-    return this.offsetToken;
+  String getEndOffsetToken() {
+    return this.endOffsetToken;
   }
 
   String getStartOffsetToken() {
     return this.startOffsetToken;
   }
 
-  void setOffsetToken(String offsetToken) {
-    this.offsetToken = offsetToken;
+  void setEndOffsetToken(String endOffsetToken) {
+    this.endOffsetToken = endOffsetToken;
   }
 
   void setStartOffsetToken(String startOffsetToken) {
@@ -164,8 +164,8 @@ public String toString() {
     return "ChannelData{"
         + "rowSequencer="
         + rowSequencer
-        + ", offsetToken='"
-        + offsetToken
+        + ", endOffsetToken='"
+        + endOffsetToken
         + ", startOffsetToken='"
         + startOffsetToken
         + '\''
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelMetadata.java
index d9043f8d7..bd072e7c3 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ChannelMetadata.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ChannelMetadata.java
@@ -16,7 +16,7 @@ class ChannelMetadata {
   private final String channelName;
   private final Long clientSequencer;
   private final Long rowSequencer;
-  @Nullable private final String offsetToken;
+  @Nullable private final String endOffsetToken;
   @Nullable private final String startOffsetToken;
 
   static Builder builder() {
@@ -47,7 +47,6 @@ Builder setOffsetToken(String offsetToken) {
       return this;
     }
 
-
     Builder setStartOffsetToken(String startOffsetToken) {
       this.startOffsetToken = startOffsetToken;
       return this;
@@ -66,7 +65,7 @@ private ChannelMetadata(Builder builder) {
     this.channelName = builder.channelName;
     this.clientSequencer = builder.clientSequencer;
     this.rowSequencer = builder.rowSequencer;
-    this.offsetToken = builder.offsetToken;
+    this.endOffsetToken = builder.offsetToken;
     this.startOffsetToken = builder.startOffsetToken;
   }
 
@@ -88,7 +87,7 @@ Long getRowSequencer() {
   @Nullable
   @JsonProperty("offset_token")
   String getOffsetToken() {
-    return this.offsetToken;
+    return this.endOffsetToken;
   }
 
   @Nullable
@@ -100,6 +99,6 @@ String getStartOffsetToken() {
   @Nullable
   @JsonProperty("end_offset_token")
   String getEndOffsetToken() {
-    return this.offsetToken;
+    return this.endOffsetToken;
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java b/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java
index 97fd0b712..a305a52f7 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FileColumnProperties.java
@@ -231,7 +231,7 @@ public boolean equals(Object o) {
     if (this == o) return true;
     if (o == null || getClass() != o.getClass()) return false;
     FileColumnProperties that = (FileColumnProperties) o;
-    return Objects.equals(columnOrdinal,that.columnOrdinal)
+    return Objects.equals(columnOrdinal, that.columnOrdinal)
         && distinctValues == that.distinctValues
         && nullCount == that.nullCount
         && maxLength == that.maxLength
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
index 94348fffc..39ec66dbb 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetFlusher.java
@@ -73,7 +73,7 @@ private SerializationResult serializeFromParquetWriteBuffers(
           ChannelMetadata.builder()
               .setOwningChannelFromContext(data.getChannelContext())
               .setRowSequencer(data.getRowSequencer())
-              .setOffsetToken(data.getOffsetToken())
+              .setOffsetToken(data.getEndOffsetToken())
               .setStartOffsetToken(data.getStartOffsetToken())
               .build();
       // Add channel metadata to the metadata list
@@ -153,7 +153,7 @@ private SerializationResult serializeFromJavaObjects(
           ChannelMetadata.builder()
               .setOwningChannelFromContext(data.getChannelContext())
               .setRowSequencer(data.getRowSequencer())
-              .setOffsetToken(data.getOffsetToken())
+              .setOffsetToken(data.getEndOffsetToken())
               .setStartOffsetToken(data.getStartOffsetToken())
               .build();
       // Add channel metadata to the metadata list
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index eeae08932..3cf7198e7 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -93,12 +93,14 @@ public void setupSchema(List<ColumnMetadata> columns) {
         addNonNullableFieldName(column.getInternalName());
       }
       this.statsMap.put(
-          column.getInternalName(), new RowBufferStats(column.getName(), column.getCollation(), column.getOrdinal()));
+          column.getInternalName(),
+          new RowBufferStats(column.getName(), column.getCollation(), column.getOrdinal()));
 
       if (onErrorOption == OpenChannelRequest.OnErrorOption.ABORT
           || onErrorOption == OpenChannelRequest.OnErrorOption.SKIP_BATCH) {
         this.tempStatsMap.put(
-            column.getInternalName(), new RowBufferStats(column.getName(), column.getCollation(), column.getOrdinal()));
+            column.getInternalName(),
+            new RowBufferStats(column.getName(), column.getCollation(), column.getOrdinal()));
       }
 
       id++;
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
index 840368764..395123f1f 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/RowBufferStats.java
@@ -33,7 +33,7 @@ class RowBufferStats {
   RowBufferStats(String columnDisplayName, String collationDefinitionString, int ordinal) {
     this.columnDisplayName = columnDisplayName;
     this.collationDefinitionString = collationDefinitionString;
-    this.ordinal= ordinal;
+    this.ordinal = ordinal;
     reset();
   }
 
@@ -54,7 +54,8 @@ void reset() {
 
   /** Create new statistics for the same column, with all calculated values set to empty */
   RowBufferStats forkEmpty() {
-    return new RowBufferStats(this.getColumnDisplayName(), this.getCollationDefinitionString(), this.getOrdinal());
+    return new RowBufferStats(
+        this.getColumnDisplayName(), this.getCollationDefinitionString(), this.getOrdinal());
   }
 
   // TODO performance test this vs in place update
@@ -68,7 +69,8 @@ static RowBufferStats getCombinedStats(RowBufferStats left, RowBufferStats right
               left.getCollationDefinitionString(), right.getCollationDefinitionString()));
     }
     RowBufferStats combined =
-        new RowBufferStats(left.columnDisplayName, left.getCollationDefinitionString(), left.getOrdinal());
+        new RowBufferStats(
+            left.columnDisplayName, left.getCollationDefinitionString(), left.getOrdinal());
 
     if (left.currentMinIntValue != null) {
       combined.addIntValue(left.currentMinIntValue);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index c756964ca..a25fd416e 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -797,12 +797,12 @@ public void testBuildAndUpload() throws Exception {
     Assert.assertEquals(2, metadataResult.getChannels().size()); // Two channels on the table
 
     Assert.assertEquals("channel1", channelMetadataResult.get(0).getChannelName());
-    Assert.assertEquals("offset1", channelMetadataResult.get(0).getOffsetToken());
+    Assert.assertEquals("offset1", channelMetadataResult.get(0).getEndOffsetToken());
     Assert.assertEquals(0L, (long) channelMetadataResult.get(0).getRowSequencer());
     Assert.assertEquals(0L, (long) channelMetadataResult.get(0).getClientSequencer());
 
     Assert.assertEquals("channel2", channelMetadataResult.get(1).getChannelName());
-    Assert.assertEquals("offset2", channelMetadataResult.get(1).getOffsetToken());
+    Assert.assertEquals("offset2", channelMetadataResult.get(1).getEndOffsetToken());
     Assert.assertEquals(10L, (long) channelMetadataResult.get(1).getRowSequencer());
     Assert.assertEquals(10L, (long) channelMetadataResult.get(1).getClientSequencer());
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index 523cacaee..e593c3d94 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -20,7 +20,6 @@
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 import org.apache.commons.codec.binary.Hex;
-import org.checkerframework.common.value.qual.IntRange;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -105,7 +104,8 @@ static List<ColumnMetadata> createSchema() {
     colChar.setLength(11);
     colChar.setScale(0);
 
-    List<ColumnMetadata> columns = Arrays.asList(
+    List<ColumnMetadata> columns =
+        Arrays.asList(
             colTinyIntCase, colTinyInt, colSmallInt, colInt, colBigInt, colDecimal, colChar);
     for (int i = 0; i < columns.size(); i++) {
       columns.get(i).setOrdinal(i + 1);
@@ -466,7 +466,8 @@ public void testFlush() {
   }
 
   private void testFlushHelper(AbstractRowBuffer<?> rowBuffer) {
-    String offsetToken = "1";
+    String startOffsetToken = "1";
+    String endOffsetToken = "2";
     Map<String, Object> row1 = new HashMap<>();
     row1.put("colTinyInt", (byte) 1);
     row1.put("\"colTinyInt\"", (byte) 1);
@@ -486,7 +487,7 @@ private void testFlushHelper(AbstractRowBuffer<?> rowBuffer) {
     row2.put("colChar", "3");
 
     InsertValidationResponse response =
-        rowBuffer.insertRows(Arrays.asList(row1, row2), offsetToken, offsetToken);
+        rowBuffer.insertRows(Arrays.asList(row1, row2), startOffsetToken, endOffsetToken);
     Assert.assertFalse(response.hasErrors());
     float bufferSize = rowBuffer.getSize();
 
@@ -494,8 +495,8 @@ private void testFlushHelper(AbstractRowBuffer<?> rowBuffer) {
     ChannelData<?> data = rowBuffer.flush(filename);
     Assert.assertEquals(2, data.getRowCount());
     Assert.assertEquals((Long) 1L, data.getRowSequencer());
-    Assert.assertEquals(offsetToken, data.getOffsetToken());
-    Assert.assertEquals(offsetToken, data.getStartOffsetToken());
+    Assert.assertEquals(startOffsetToken, data.getStartOffsetToken());
+    Assert.assertEquals(endOffsetToken, data.getEndOffsetToken());
     Assert.assertEquals(bufferSize, data.getBufferSize(), 0);
 
     final ParquetChunkData chunkData = (ParquetChunkData) data.getVectors();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 6d1697cff..814ce77c1 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -585,7 +585,7 @@ public void testInsertRow() {
     Assert.assertEquals(3, data.getRowCount());
     Assert.assertEquals((Long) 1L, data.getRowSequencer());
     Assert.assertEquals(1, ((ChannelData<ParquetChunkData>) data).getVectors().rows.get(0).size());
-    Assert.assertEquals("3", data.getOffsetToken());
+    Assert.assertEquals("3", data.getEndOffsetToken());
     Assert.assertEquals("1", data.getStartOffsetToken());
     Assert.assertTrue(data.getBufferSize() > 0);
     Assert.assertTrue(insertStartTimeInMs <= data.getMinMaxInsertTimeInMs().getFirst());

From 9cbbd1c8e5c49d307b83a144c61dfbcbbfd6f254 Mon Sep 17 00:00:00 2001
From: Jay Patel <jay.patel@snowflake.com>
Date: Wed, 21 Feb 2024 10:51:18 -0800
Subject: [PATCH 324/356] =?UTF-8?q?SNOW-987122=20Upgrade=20JDBC=20to=203.1?=
 =?UTF-8?q?4.5=20and=20Catch=20new=20exception=20type=20for=20r=E2=80=A6?=
 =?UTF-8?q?=20(#677)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* SNOW-987122 Upgrade JDBC to 3.14.5 and Catch new exception type for renewing expired S3 token

* Modify enforcer rule for duplicate classes

* Fix test and set UTC TZ

* Temporary fix for failing long variant, object and array test

* Do not commit this - Long running test

* Remove isCredentialExpired exception and refresh metadata on first failure of upload

* Long running fips e2e test- lukas suggestion

* Revert long running tests

* Remove unused function and formatter

* Add stacktrace in logs
---
 README.md                                     |  4 +-
 pom.xml                                       |  5 +-
 public_pom.xml                                |  2 +-
 .../internal/StreamingIngestStage.java        | 41 ++++------
 .../internal/DataValidationUtilTest.java      |  2 +
 .../internal/StreamingIngestStageTest.java    | 78 +++++++++++++++----
 .../datatypes/AbstractDataTypeTest.java       | 45 ++++++++---
 .../internal/datatypes/SemiStructuredIT.java  | 20 ++++-
 8 files changed, 136 insertions(+), 61 deletions(-)

diff --git a/README.md b/README.md
index fcf935d51..e0cd0d8b1 100644
--- a/README.md
+++ b/README.md
@@ -15,7 +15,7 @@ authentication. Currently, we support ingestion through the following APIs:
 
 The Snowflake Ingest Service SDK depends on the following libraries:
 
-* snowflake-jdbc (3.13.30 to 3.13.33) 
+* snowflake-jdbc (3.13.30 to 3.14.5) 
 * slf4j-api
 * com.github.luben:zstd-jni (1.5.0-1)
 
@@ -24,7 +24,7 @@ using a build system, please make sure these dependencies are on the classpath.
 
 # Prerequisites
 
-**If your project depends on the Snowflake JDBC driver, as well, please make sure the JDBC driver version is 3.13.30 to 3.13.33. JDBC driver version 3.14.0 or higher is currently not supported. **
+**If your project depends on the Snowflake JDBC driver, as well, please make sure the JDBC driver version is 3.13.30 to 3.14.5.
 
 ## Java 8+
 
diff --git a/pom.xml b/pom.xml
index 47a1663d9..8992b1264 100644
--- a/pom.xml
+++ b/pom.xml
@@ -65,7 +65,7 @@
     <shadeBase>net.snowflake.ingest.internal</shadeBase>
     <slf4j.version>1.7.36</slf4j.version>
     <snappy.version>1.1.10.4</snappy.version>
-    <snowjdbc.version>3.13.30</snowjdbc.version>
+    <snowjdbc.version>3.14.5</snowjdbc.version>
     <yetus.version>0.13.0</yetus.version>
   </properties>
 
@@ -711,6 +711,9 @@
             <configuration>
               <rules>
                 <banDuplicateClasses>
+                  <ignoreClasses>
+                    <ignoreClass>META-INF/versions/*/org/bouncycastle/*</ignoreClass>
+                  </ignoreClasses>
                   <findAllDuplicates>true</findAllDuplicates>
                   <ignoreWhenIdentical>true</ignoreWhenIdentical>
                 </banDuplicateClasses>
diff --git a/public_pom.xml b/public_pom.xml
index fb03684cf..fa8c7dd4a 100644
--- a/public_pom.xml
+++ b/public_pom.xml
@@ -39,7 +39,7 @@
         <dependency>
             <groupId>net.snowflake</groupId>
             <artifactId>snowflake-jdbc</artifactId>
-            <version>3.13.30</version>
+            <version>3.14.5</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index 443097550..e8e56f383 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -4,12 +4,12 @@
 
 package net.snowflake.ingest.streaming.internal;
 
-import static net.snowflake.client.core.Constants.CLOUD_STORAGE_CREDENTIALS_EXPIRED;
 import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CLIENT_CONFIGURE;
 import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
 import static net.snowflake.ingest.utils.Constants.CLIENT_CONFIGURE_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
 import static net.snowflake.ingest.utils.HttpUtil.generateProxyPropertiesForJDBC;
+import static net.snowflake.ingest.utils.Utils.getStackTrace;
 
 import com.google.common.annotations.VisibleForTesting;
 import java.io.ByteArrayInputStream;
@@ -34,7 +34,6 @@
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode;
-import net.snowflake.client.jdbc.internal.google.cloud.storage.StorageException;
 import net.snowflake.ingest.connection.IngestResponseException;
 import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.utils.ErrorCode;
@@ -193,6 +192,13 @@ private void putRemote(String fullFilePath, byte[] data, int retryCount)
               .setDestFileName(fullFilePath)
               .build());
     } catch (Exception e) {
+      if (retryCount == 0) {
+        // for the first exception, we always perform a metadata refresh.
+        logger.logInfo(
+            "Stage metadata need to be refreshed due to upload error: {} on first retry attempt",
+            e.getMessage());
+        this.refreshSnowflakeMetadata();
+      }
       if (retryCount >= maxUploadRetries) {
         logger.logError(
             "Failed to upload to stage, retry attempts exhausted ({}), client={}, message={}",
@@ -201,39 +207,18 @@ private void putRemote(String fullFilePath, byte[] data, int retryCount)
             e.getMessage());
         throw new SFException(e, ErrorCode.IO_ERROR);
       }
-
-      if (isCredentialsExpiredException(e)) {
-        logger.logInfo(
-            "Stage metadata need to be refreshed due to upload error: {}", e.getMessage());
-        this.refreshSnowflakeMetadata();
-      }
       retryCount++;
       StreamingIngestUtils.sleepForRetry(retryCount);
       logger.logInfo(
-          "Retrying upload, attempt {}/{} {}", retryCount, maxUploadRetries, e.getMessage());
+          "Retrying upload, attempt {}/{} msg: {}, stackTrace:{}",
+          retryCount,
+          maxUploadRetries,
+          e.getMessage(),
+          getStackTrace(e));
       this.putRemote(fullFilePath, data, retryCount);
     }
   }
 
-  /**
-   * @return Whether the passed exception means that credentials expired and the stage metadata
-   *     should be refreshed from Snowflake. The reasons for refresh is SnowflakeSQLException with
-   *     error code 240001 (thrown by the JDBC driver) or GCP StorageException with HTTP status 401.
-   */
-  static boolean isCredentialsExpiredException(Exception e) {
-    if (e == null || e.getClass() == null) {
-      return false;
-    }
-
-    if (e instanceof SnowflakeSQLException) {
-      return ((SnowflakeSQLException) e).getErrorCode() == CLOUD_STORAGE_CREDENTIALS_EXPIRED;
-    } else if (e instanceof StorageException) {
-      return ((StorageException) e).getCode() == 401;
-    }
-
-    return false;
-  }
-
   SnowflakeFileTransferMetadataWithAge refreshSnowflakeMetadata()
       throws SnowflakeSQLException, IOException {
     logger.logInfo("Refresh Snowflake metadata, client={}", clientName);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
index 9467899e2..8ab22619f 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/DataValidationUtilTest.java
@@ -44,6 +44,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.TimeZone;
 import net.snowflake.ingest.utils.ErrorCode;
 import net.snowflake.ingest.utils.SFException;
 import org.apache.commons.codec.DecoderException;
@@ -269,6 +270,7 @@ public void testValidateAndParseTimestamp() throws ParseException {
 
     // Test integer-stored time and scale guessing
     SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
+    df.setTimeZone(TimeZone.getTimeZone("UTC"));
     assertEquals(
         BigInteger.valueOf(df.parse("1971-01-01 00:00:00.001").getTime())
             .multiply(BigInteger.valueOf(1000000)),
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
index a14f1c46b..1ba9f98df 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java
@@ -1,7 +1,6 @@
 package net.snowflake.ingest.streaming.internal;
 
 import static net.snowflake.client.core.Constants.CLOUD_STORAGE_CREDENTIALS_EXPIRED;
-import static net.snowflake.ingest.streaming.internal.StreamingIngestStage.isCredentialsExpiredException;
 import static net.snowflake.ingest.utils.HttpUtil.HTTP_PROXY_PASSWORD;
 import static net.snowflake.ingest.utils.HttpUtil.HTTP_PROXY_USER;
 import static net.snowflake.ingest.utils.HttpUtil.NON_PROXY_HOSTS;
@@ -40,7 +39,6 @@
 import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
-import net.snowflake.client.jdbc.internal.google.cloud.storage.StorageException;
 import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder;
 import net.snowflake.ingest.TestUtils;
 import net.snowflake.ingest.connection.RequestBuilder;
@@ -486,20 +484,66 @@ public void testShouldBypassProxy() {
   }
 
   @Test
-  public void testIsCredentialExpiredException() {
-    Assert.assertTrue(
-        isCredentialsExpiredException(
-            new SnowflakeSQLException("Error", CLOUD_STORAGE_CREDENTIALS_EXPIRED)));
-    Assert.assertTrue(isCredentialsExpiredException(new StorageException(401, "unauthorized")));
-
-    Assert.assertFalse(isCredentialsExpiredException(new StorageException(400, "bad request")));
-    Assert.assertFalse(isCredentialsExpiredException(null));
-    Assert.assertFalse(isCredentialsExpiredException(new RuntimeException()));
-    Assert.assertFalse(
-        isCredentialsExpiredException(
-            new RuntimeException(String.valueOf(CLOUD_STORAGE_CREDENTIALS_EXPIRED))));
-    Assert.assertFalse(
-        isCredentialsExpiredException(
-            new SnowflakeSQLException("Error", CLOUD_STORAGE_CREDENTIALS_EXPIRED + 1)));
+  public void testRefreshMetadataOnFirstPutException() throws Exception {
+    int maxUploadRetryCount = 2;
+    JsonNode exampleJson = mapper.readTree(exampleRemoteMeta);
+    SnowflakeFileTransferMetadataV1 originalMetadata =
+        (SnowflakeFileTransferMetadataV1)
+            SnowflakeFileTransferAgent.getFileTransferMetadatas(exampleJson).get(0);
+
+    byte[] dataBytes = "Hello Upload".getBytes(StandardCharsets.UTF_8);
+
+    StreamingIngestStage stage =
+        new StreamingIngestStage(
+            true,
+            "role",
+            null,
+            null,
+            "clientName",
+            new StreamingIngestStage.SnowflakeFileTransferMetadataWithAge(
+                originalMetadata, Optional.of(System.currentTimeMillis())),
+            maxUploadRetryCount);
+    PowerMockito.mockStatic(SnowflakeFileTransferAgent.class);
+    SnowflakeSQLException e =
+        new SnowflakeSQLException(
+            "Fake bad creds", CLOUD_STORAGE_CREDENTIALS_EXPIRED, "S3 credentials have expired");
+    PowerMockito.doAnswer(
+            new org.mockito.stubbing.Answer() {
+              private boolean firstInvocation = true;
+
+              public Object answer(org.mockito.invocation.InvocationOnMock invocation)
+                  throws Throwable {
+                if (firstInvocation) {
+                  firstInvocation = false;
+                  throw e; // Throw the exception only for the first invocation
+                }
+                return null; // Do nothing on subsequent invocations
+              }
+            })
+        .when(SnowflakeFileTransferAgent.class);
+    SnowflakeFileTransferAgent.uploadWithoutConnection(Mockito.any());
+    final ArgumentCaptor<SnowflakeFileTransferConfig> captor =
+        ArgumentCaptor.forClass(SnowflakeFileTransferConfig.class);
+
+    stage.putRemote("test/path", dataBytes);
+
+    PowerMockito.verifyStatic(SnowflakeFileTransferAgent.class, times(maxUploadRetryCount));
+    SnowflakeFileTransferAgent.uploadWithoutConnection(captor.capture());
+    SnowflakeFileTransferConfig capturedConfig = captor.getValue();
+
+    Assert.assertFalse(capturedConfig.getRequireCompress());
+    Assert.assertEquals(OCSPMode.FAIL_OPEN, capturedConfig.getOcspMode());
+
+    SnowflakeFileTransferMetadataV1 capturedMetadata =
+        (SnowflakeFileTransferMetadataV1) capturedConfig.getSnowflakeFileTransferMetadata();
+    Assert.assertEquals("test/path", capturedMetadata.getPresignedUrlFileName());
+    Assert.assertEquals(originalMetadata.getCommandType(), capturedMetadata.getCommandType());
+    Assert.assertEquals(originalMetadata.getPresignedUrl(), capturedMetadata.getPresignedUrl());
+    Assert.assertEquals(
+        originalMetadata.getStageInfo().getStageType(),
+        capturedMetadata.getStageInfo().getStageType());
+
+    InputStream capturedInput = capturedConfig.getUploadStream();
+    Assert.assertEquals("Hello Upload", IOUtils.toString(capturedInput));
   }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
index 4d510acf6..ec3017ad5 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/AbstractDataTypeTest.java
@@ -319,6 +319,21 @@ <STREAMING_INGEST_WRITE> void assertVariant(
       String expectedValue,
       String expectedType)
       throws Exception {
+    assertVariant(
+        dataType,
+        streamingIngestWriteValue,
+        expectedValue,
+        expectedType,
+        false /*Not max variant or max array*/);
+  }
+
+  <STREAMING_INGEST_WRITE> void assertVariant(
+      String dataType,
+      STREAMING_INGEST_WRITE streamingIngestWriteValue,
+      String expectedValue,
+      String expectedType,
+      boolean isAtMaxValueForDataType)
+      throws Exception {
 
     String tableName = createTable(dataType);
     String offsetToken = UUID.randomUUID().toString();
@@ -328,24 +343,36 @@ <STREAMING_INGEST_WRITE> void assertVariant(
     channel.insertRow(createStreamingIngestRow(streamingIngestWriteValue), offsetToken);
     TestUtils.waitForOffset(channel, offsetToken);
 
-    String query =
-        String.format(
-            "select %s, typeof(%s) from %s", VALUE_COLUMN_NAME, VALUE_COLUMN_NAME, tableName);
+    String query;
+    // if added row value is using max possible value, we will not verify its returned type since
+    // JDBC doesnt parse it properly because of Jackson Databind version 2.15
+    // we will only verify type of it.
+    // Check this: https://github.com/snowflakedb/snowflake-sdks-drivers-issues-teamwork/issues/819
+    // TODO:: SNOW-1051731
+    if (isAtMaxValueForDataType) {
+      query = String.format("select typeof(%s) from %s", VALUE_COLUMN_NAME, tableName);
+    } else {
+      query =
+          String.format(
+              "select typeof(%s), %s  from %s", VALUE_COLUMN_NAME, VALUE_COLUMN_NAME, tableName);
+    }
     ResultSet resultSet = conn.createStatement().executeQuery(query);
     int counter = 0;
     String value = null;
     String typeof = null;
     while (resultSet.next()) {
       counter++;
-      value = resultSet.getString(1);
-      typeof = resultSet.getString(2);
+      typeof = resultSet.getString(1);
+      if (!isAtMaxValueForDataType) value = resultSet.getString(2);
     }
 
     Assert.assertEquals(1, counter);
-    if (expectedValue == null) {
-      Assert.assertNull(value);
-    } else {
-      Assert.assertEquals(objectMapper.readTree(expectedValue), objectMapper.readTree(value));
+    if (!isAtMaxValueForDataType) {
+      if (expectedValue == null) {
+        Assert.assertNull(value);
+      } else {
+        Assert.assertEquals(objectMapper.readTree(expectedValue), objectMapper.readTree(value));
+      }
     }
     Assert.assertEquals(expectedType, typeof);
     migrateTable(tableName); // migration should always succeed
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
index 49e994180..a3c9d2365 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/datatypes/SemiStructuredIT.java
@@ -171,17 +171,31 @@ public void testVariant() throws Exception {
     assertVariant("VARIANT", null, null, null);
   }
 
+  /**
+   * For JDBC version > 3.13.3 we dont verify the value returned from max variant, max array and max
+   * object because of
+   * https://github.com/snowflakedb/snowflake-sdks-drivers-issues-teamwork/issues/819
+   *
+   * @throws Exception
+   */
   @Test
   public void testMaxVariantAndObject() throws Exception {
     String maxObject = createLargeVariantObject(MAX_ALLOWED_LENGTH);
-    assertVariant("VARIANT", maxObject, maxObject, "OBJECT");
-    assertVariant("OBJECT", maxObject, maxObject, "OBJECT");
+    assertVariant("VARIANT", maxObject, maxObject, "OBJECT", true);
+    assertVariant("OBJECT", maxObject, maxObject, "OBJECT", true);
   }
 
+  /**
+   * For JDBC version > 3.13.3 we dont verify the value returned from max variant, max array and max
+   * object because of
+   * https://github.com/snowflakedb/snowflake-sdks-drivers-issues-teamwork/issues/819
+   *
+   * @throws Exception
+   */
   @Test
   public void testMaxArray() throws Exception {
     String maxArray = "[" + createLargeVariantObject(MAX_ALLOWED_LENGTH - 2) + "]";
-    assertVariant("ARRAY", maxArray, maxArray, "ARRAY");
+    assertVariant("ARRAY", maxArray, maxArray, "ARRAY", true);
   }
 
   @Test

From 61e1b287d78ec931e8ae807c9d42ade00523660d Mon Sep 17 00:00:00 2001
From: Purujit Saha <purujit.saha@snowflake.com>
Date: Wed, 21 Feb 2024 15:42:16 -0800
Subject: [PATCH 325/356] Allow clients to drop channel on close or blindly
 (#657)

* Add client side support for upcoming drop channel API

* Address review comments

* Add copyright blurb
---
 .../connection/ServiceResponseHandler.java    |  1 +
 .../ingest/streaming/DropChannelRequest.java  | 89 +++++++++++++++++++
 .../SnowflakeStreamingIngestChannel.java      |  8 ++
 .../SnowflakeStreamingIngestClient.java       | 12 +++
 .../internal/DropChannelResponse.java         | 72 +++++++++++++++
 .../internal/DropChannelVersionRequest.java   | 23 +++++
 ...owflakeStreamingIngestChannelInternal.java | 15 ++++
 ...nowflakeStreamingIngestClientInternal.java | 65 ++++++++++++++
 .../net/snowflake/ingest/utils/Constants.java |  1 +
 .../net/snowflake/ingest/utils/ErrorCode.java |  3 +-
 .../SnowflakeStreamingIngestChannelTest.java  | 71 +++++++++++++++
 .../SnowflakeStreamingIngestClientTest.java   | 49 ++++++++++
 .../streaming/internal/StreamingIngestIT.java | 35 ++++++++
 13 files changed, 443 insertions(+), 1 deletion(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/DropChannelRequest.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/DropChannelResponse.java
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/DropChannelVersionRequest.java

diff --git a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java
index f6c7a6c1b..bf20afda8 100644
--- a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java
+++ b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java
@@ -39,6 +39,7 @@ public enum ApiName {
     INSERT_REPORT("GET"),
     LOAD_HISTORY_SCAN("GET"),
     STREAMING_OPEN_CHANNEL("POST"),
+    STREAMING_DROP_CHANNEL("POST"),
     STREAMING_CHANNEL_STATUS("POST"),
     STREAMING_REGISTER_BLOB("POST"),
     STREAMING_CLIENT_CONFIGURE("POST");
diff --git a/src/main/java/net/snowflake/ingest/streaming/DropChannelRequest.java b/src/main/java/net/snowflake/ingest/streaming/DropChannelRequest.java
new file mode 100644
index 000000000..3bc096c16
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/DropChannelRequest.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2024 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming;
+
+import net.snowflake.ingest.utils.Utils;
+
+/** A class that is used to drop a {@link SnowflakeStreamingIngestChannel} */
+public class DropChannelRequest {
+  // Name of the channel
+  private final String channelName;
+
+  // Name of the database that the channel belongs to
+  private final String dbName;
+
+  // Name of the schema that the channel belongs to
+  private final String schemaName;
+
+  // Name of the table that the channel belongs to
+  private final String tableName;
+
+  public static DropChannelRequestBuilder builder(String channelName) {
+    return new DropChannelRequestBuilder(channelName);
+  }
+
+  /** A builder class to build a DropChannelRequest */
+  public static class DropChannelRequestBuilder {
+    private final String channelName;
+    private String dbName;
+    private String schemaName;
+    private String tableName;
+
+    public DropChannelRequestBuilder(String channelName) {
+      this.channelName = channelName;
+    }
+
+    public DropChannelRequestBuilder setDBName(String dbName) {
+      this.dbName = dbName;
+      return this;
+    }
+
+    public DropChannelRequestBuilder setSchemaName(String schemaName) {
+      this.schemaName = schemaName;
+      return this;
+    }
+
+    public DropChannelRequestBuilder setTableName(String tableName) {
+      this.tableName = tableName;
+      return this;
+    }
+
+    public DropChannelRequest build() {
+      return new DropChannelRequest(this);
+    }
+  }
+
+  public DropChannelRequest(DropChannelRequestBuilder builder) {
+    Utils.assertStringNotNullOrEmpty("channel name", builder.channelName);
+    Utils.assertStringNotNullOrEmpty("database name", builder.dbName);
+    Utils.assertStringNotNullOrEmpty("schema name", builder.schemaName);
+    Utils.assertStringNotNullOrEmpty("table name", builder.tableName);
+
+    this.channelName = builder.channelName;
+    this.dbName = builder.dbName;
+    this.schemaName = builder.schemaName;
+    this.tableName = builder.tableName;
+  }
+
+  public String getDBName() {
+    return this.dbName;
+  }
+
+  public String getSchemaName() {
+    return this.schemaName;
+  }
+
+  public String getTableName() {
+    return this.tableName;
+  }
+
+  public String getChannelName() {
+    return this.channelName;
+  }
+
+  public String getFullyQualifiedTableName() {
+    return String.format("%s.%s.%s", this.dbName, this.schemaName, this.tableName);
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
index 2ab48b53f..b8687358e 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
@@ -79,6 +79,14 @@ public interface SnowflakeStreamingIngestChannel {
    */
   CompletableFuture<Void> close();
 
+  /**
+   * Close the channel, this function will make sure all the data in this channel is committed
+   *
+   * @param drop if true, the channel will be dropped after all data is successfully committed.
+   * @return a completable future which will be completed when the channel is closed
+   */
+  CompletableFuture<Void> close(boolean drop);
+
   /**
    * Insert one row into the channel, the row is represented using Map where the key is column name
    * and the value is a row of data. The following table summarizes supported value types and their
diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
index 3ae4a83d1..8b65b5cae 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
@@ -25,6 +25,18 @@ public interface SnowflakeStreamingIngestClient extends AutoCloseable {
    */
   SnowflakeStreamingIngestChannel openChannel(OpenChannelRequest request);
 
+  /**
+   * Drop the specified channel on the server using a {@link DropChannelRequest}
+   *
+   * <p>Note that this call will blindly drop the latest version of the channel and any pending data
+   * will be lost. Also see {@link SnowflakeStreamingIngestChannel#close(boolean)} to drop channels
+   * on close. That approach will drop the local version of the channel and if the channel has been
+   * concurrently reopened by another client, that version of the channel won't be affected.
+   *
+   * @param request the drop channel request
+   */
+  void dropChannel(DropChannelRequest request);
+
   /**
    * Get the client name
    *
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DropChannelResponse.java b/src/main/java/net/snowflake/ingest/streaming/internal/DropChannelResponse.java
new file mode 100644
index 000000000..fce91f1d2
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DropChannelResponse.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2024 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/** Response for a {@link net.snowflake.ingest.streaming.DropChannelRequest}. */
+class DropChannelResponse extends StreamingIngestResponse {
+  private Long statusCode;
+  private String message;
+  private String dbName;
+  private String schemaName;
+  private String tableName;
+  private String channelName;
+
+  @JsonProperty("status_code")
+  void setStatusCode(Long statusCode) {
+    this.statusCode = statusCode;
+  }
+
+  @Override
+  Long getStatusCode() {
+    return this.statusCode;
+  }
+
+  @JsonProperty("message")
+  void setMessage(String message) {
+    this.message = message;
+  }
+
+  String getMessage() {
+    return this.message;
+  }
+
+  @JsonProperty("database")
+  void setDBName(String dbName) {
+    this.dbName = dbName;
+  }
+
+  String getDBName() {
+    return this.dbName;
+  }
+
+  @JsonProperty("schema")
+  void setSchemaName(String schemaName) {
+    this.schemaName = schemaName;
+  }
+
+  String getSchemaName() {
+    return this.schemaName;
+  }
+
+  @JsonProperty("table")
+  void setTableName(String tableName) {
+    this.tableName = tableName;
+  }
+
+  String getTableName() {
+    return this.tableName;
+  }
+
+  @JsonProperty("channel")
+  void setChannelName(String channelName) {
+    this.channelName = channelName;
+  }
+
+  String getChannelName() {
+    return this.channelName;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DropChannelVersionRequest.java b/src/main/java/net/snowflake/ingest/streaming/internal/DropChannelVersionRequest.java
new file mode 100644
index 000000000..abc6b8067
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/DropChannelVersionRequest.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2024 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming.internal;
+
+import net.snowflake.ingest.streaming.DropChannelRequest;
+
+/**
+ * Same as DropChannelRequest but allows specifying a client sequencer to drop a specific version.
+ */
+class DropChannelVersionRequest extends DropChannelRequest {
+  private final Long clientSequencer;
+
+  public DropChannelVersionRequest(DropChannelRequestBuilder builder, long clientSequencer) {
+    super(builder);
+    this.clientSequencer = clientSequencer;
+  }
+
+  Long getClientSequencer() {
+    return this.clientSequencer;
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 695f5632b..2f4a7678d 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -20,6 +20,7 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
+import net.snowflake.ingest.streaming.DropChannelRequest;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
@@ -267,6 +268,11 @@ CompletableFuture<Void> flush(boolean closing) {
    */
   @Override
   public CompletableFuture<Void> close() {
+    return this.close(false);
+  }
+
+  @Override
+  public CompletableFuture<Void> close(boolean drop) {
     checkValidation();
 
     if (isClosed()) {
@@ -292,6 +298,15 @@ public CompletableFuture<Void> close() {
                         .map(SnowflakeStreamingIngestChannelInternal::getFullyQualifiedName)
                         .collect(Collectors.toList()));
               }
+              if (drop) {
+                DropChannelRequest.DropChannelRequestBuilder builder =
+                    DropChannelRequest.builder(this.getChannelContext().getName())
+                        .setDBName(this.getDBName())
+                        .setTableName(this.getTableName())
+                        .setSchemaName(this.getSchemaName());
+                this.owningClient.dropChannel(
+                    new DropChannelVersionRequest(builder, this.getChannelSequencer()));
+              }
             });
   }
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 898642fd1..0dde314f4 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -5,6 +5,7 @@
 package net.snowflake.ingest.streaming.internal;
 
 import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_CHANNEL_STATUS;
+import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_DROP_CHANNEL;
 import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_OPEN_CHANNEL;
 import static net.snowflake.ingest.connection.ServiceResponseHandler.ApiName.STREAMING_REGISTER_BLOB;
 import static net.snowflake.ingest.streaming.internal.StreamingIngestUtils.executeWithRetries;
@@ -12,6 +13,7 @@
 import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.COMMIT_MAX_RETRY_COUNT;
 import static net.snowflake.ingest.utils.Constants.COMMIT_RETRY_INTERVAL_IN_MS;
+import static net.snowflake.ingest.utils.Constants.DROP_CHANNEL_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.ENABLE_TELEMETRY_TO_SF;
 import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
 import static net.snowflake.ingest.utils.Constants.OPEN_CHANNEL_ENDPOINT;
@@ -63,6 +65,7 @@
 import net.snowflake.ingest.connection.OAuthCredential;
 import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.connection.TelemetryService;
+import net.snowflake.ingest.streaming.DropChannelRequest;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
@@ -367,6 +370,68 @@ public SnowflakeStreamingIngestChannelInternal<?> openChannel(OpenChannelRequest
     }
   }
 
+  @Override
+  public void dropChannel(DropChannelRequest request) {
+    if (isClosed) {
+      throw new SFException(ErrorCode.CLOSED_CLIENT);
+    }
+
+    logger.logDebug(
+        "Drop channel request start, channel={}, table={}, client={}",
+        request.getChannelName(),
+        request.getFullyQualifiedTableName(),
+        getName());
+
+    try {
+      Map<Object, Object> payload = new HashMap<>();
+      payload.put(
+          "request_id", this.flushService.getClientPrefix() + "_" + counter.getAndIncrement());
+      payload.put("channel", request.getChannelName());
+      payload.put("table", request.getTableName());
+      payload.put("database", request.getDBName());
+      payload.put("schema", request.getSchemaName());
+      payload.put("role", this.role);
+      Long clientSequencer = null;
+      if (request instanceof DropChannelVersionRequest) {
+        clientSequencer = ((DropChannelVersionRequest) request).getClientSequencer();
+        if (clientSequencer != null) {
+          payload.put("client_sequencer", clientSequencer);
+        }
+      }
+
+      DropChannelResponse response =
+          executeWithRetries(
+              DropChannelResponse.class,
+              DROP_CHANNEL_ENDPOINT,
+              payload,
+              "drop channel",
+              STREAMING_DROP_CHANNEL,
+              httpClient,
+              requestBuilder);
+
+      // Check for Snowflake specific response code
+      if (response.getStatusCode() != RESPONSE_SUCCESS) {
+        logger.logDebug(
+            "Drop channel request failed, channel={}, table={}, client={}, message={}",
+            request.getChannelName(),
+            request.getFullyQualifiedTableName(),
+            getName(),
+            response.getMessage());
+        throw new SFException(ErrorCode.DROP_CHANNEL_FAILURE, response.getMessage());
+      }
+
+      logger.logInfo(
+          "Drop channel request succeeded, channel={}, table={}, clientSequencer={} client={}",
+          request.getChannelName(),
+          request.getFullyQualifiedTableName(),
+          clientSequencer,
+          getName());
+
+    } catch (IOException | IngestResponseException e) {
+      throw new SFException(e, ErrorCode.DROP_CHANNEL_FAILURE, e.getMessage());
+    }
+  }
+
   /**
    * Return the latest committed/persisted offset token for all channels
    *
diff --git a/src/main/java/net/snowflake/ingest/utils/Constants.java b/src/main/java/net/snowflake/ingest/utils/Constants.java
index 404ec3851..134328668 100644
--- a/src/main/java/net/snowflake/ingest/utils/Constants.java
+++ b/src/main/java/net/snowflake/ingest/utils/Constants.java
@@ -62,6 +62,7 @@ public class Constants {
   // Channel level constants
   public static final String CHANNEL_STATUS_ENDPOINT = "/v1/streaming/channels/status/";
   public static final String OPEN_CHANNEL_ENDPOINT = "/v1/streaming/channels/open/";
+  public static final String DROP_CHANNEL_ENDPOINT = "/v1/streaming/channels/drop/";
   public static final String REGISTER_BLOB_ENDPOINT = "/v1/streaming/channels/write/blobs/";
 
   public enum WriteMode {
diff --git a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
index b212d7244..46c11a485 100644
--- a/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
+++ b/src/main/java/net/snowflake/ingest/utils/ErrorCode.java
@@ -41,7 +41,8 @@ public enum ErrorCode {
   OAUTH_REFRESH_TOKEN_ERROR("0033"),
   INVALID_CONFIG_PARAMETER("0034"),
   MAX_BATCH_SIZE_EXCEEDED("0035"),
-  CRYPTO_PROVIDER_ERROR("0036");
+  CRYPTO_PROVIDER_ERROR("0036"),
+  DROP_CHANNEL_FAILURE("0037");
 
   public static final String errorMessageResource = "net.snowflake.ingest.ingest_error_messages";
 
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
index 814ce77c1..780563d28 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java
@@ -7,6 +7,7 @@
 import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS;
 import static net.snowflake.ingest.utils.Constants.ROLE;
 import static net.snowflake.ingest.utils.Constants.USER;
+import static org.mockito.ArgumentMatchers.argThat;
 
 import java.security.KeyPair;
 import java.security.PrivateKey;
@@ -814,6 +815,76 @@ public void testClose() throws Exception {
 
     // Calling close again on closed channel shouldn't fail
     channel.close().get();
+    Mockito.verify(client, Mockito.times(0)).dropChannel(Mockito.any());
+  }
+
+  @Test
+  public void testDropOnClose() throws Exception {
+    SnowflakeStreamingIngestClientInternal<?> client =
+        Mockito.spy(new SnowflakeStreamingIngestClientInternal<>("client"));
+    SnowflakeStreamingIngestChannelInternal channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
+            "channel",
+            "db",
+            "schema",
+            "table",
+            "0",
+            1L,
+            0L,
+            client,
+            "key",
+            1234L,
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
+    ChannelsStatusResponse response = new ChannelsStatusResponse();
+    response.setStatusCode(0L);
+    response.setMessage("Success");
+    response.setChannels(new ArrayList<>());
+
+    Mockito.doReturn(response).when(client).getChannelsStatus(Mockito.any());
+
+    Assert.assertFalse(channel.isClosed());
+    Mockito.doNothing().when(client).dropChannel(Mockito.any());
+    channel.close(true).get();
+    Assert.assertTrue(channel.isClosed());
+    Mockito.verify(client, Mockito.times(1))
+        .dropChannel(
+            argThat(
+                (DropChannelVersionRequest req) ->
+                    req.getChannelName().equals(channel.getName())
+                        && req.getClientSequencer().equals(channel.getChannelSequencer())));
+  }
+
+  @Test
+  public void testDropOnCloseInvalidChannel() throws Exception {
+    SnowflakeStreamingIngestClientInternal<?> client =
+        Mockito.spy(new SnowflakeStreamingIngestClientInternal<>("client"));
+    SnowflakeStreamingIngestChannelInternal channel =
+        new SnowflakeStreamingIngestChannelInternal<>(
+            "channel",
+            "db",
+            "schema",
+            "table",
+            "0",
+            1L,
+            0L,
+            client,
+            "key",
+            1234L,
+            OpenChannelRequest.OnErrorOption.CONTINUE,
+            UTC);
+    ChannelsStatusResponse response = new ChannelsStatusResponse();
+    response.setStatusCode(0L);
+    response.setMessage("Success");
+    response.setChannels(new ArrayList<>());
+
+    Mockito.doReturn(response).when(client).getChannelsStatus(Mockito.any());
+
+    Assert.assertFalse(channel.isClosed());
+    channel.invalidate("test");
+    Mockito.doNothing().when(client).dropChannel(Mockito.any());
+    Assert.assertThrows(SFException.class, () -> channel.close(true).get());
+    Mockito.verify(client, Mockito.never()).dropChannel(Mockito.any());
   }
 
   @Test
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index 1107aad89..dda34d83d 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -3,6 +3,7 @@
 import static java.time.ZoneOffset.UTC;
 import static net.snowflake.ingest.utils.Constants.ACCOUNT_URL;
 import static net.snowflake.ingest.utils.Constants.CHANNEL_STATUS_ENDPOINT;
+import static net.snowflake.ingest.utils.Constants.DROP_CHANNEL_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.MAX_STREAMING_INGEST_API_CHANNEL_RETRY;
 import static net.snowflake.ingest.utils.Constants.PRIVATE_KEY;
 import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
@@ -19,6 +20,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
 import java.io.StringWriter;
+import java.nio.charset.Charset;
 import java.security.KeyPair;
 import java.security.PrivateKey;
 import java.time.ZoneOffset;
@@ -43,6 +45,7 @@
 import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
 import net.snowflake.ingest.TestUtils;
 import net.snowflake.ingest.connection.RequestBuilder;
+import net.snowflake.ingest.streaming.DropChannelRequest;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
@@ -63,6 +66,7 @@
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
+import org.mockito.ArgumentMatchers;
 import org.mockito.Mockito;
 
 public class SnowflakeStreamingIngestClientTest {
@@ -368,6 +372,51 @@ public void testGetChannelsStatusWithRequest() throws Exception {
             objectMapper.writeValueAsString(request), CHANNEL_STATUS_ENDPOINT, "channel status");
   }
 
+  @Test
+  public void testDropChannel() throws Exception {
+    DropChannelResponse response = new DropChannelResponse();
+    response.setStatusCode(RESPONSE_SUCCESS);
+    response.setMessage("dropped");
+    String responseString = objectMapper.writeValueAsString(response);
+
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+    CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class);
+    StatusLine statusLine = Mockito.mock(StatusLine.class);
+    HttpEntity httpEntity = Mockito.mock(HttpEntity.class);
+    when(statusLine.getStatusCode()).thenReturn(200);
+    when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    when(httpResponse.getEntity()).thenReturn(httpEntity);
+    when(httpEntity.getContent())
+        .thenReturn(IOUtils.toInputStream(responseString, Charset.defaultCharset()));
+    when(httpClient.execute(Mockito.any())).thenReturn(httpResponse);
+
+    RequestBuilder requestBuilder =
+        Mockito.spy(
+            new RequestBuilder(TestUtils.getHost(), TestUtils.getUser(), TestUtils.getKeyPair()));
+    SnowflakeStreamingIngestClientInternal<?> client =
+        new SnowflakeStreamingIngestClientInternal<>(
+            "client",
+            new SnowflakeURL("snowflake.dev.local:8082"),
+            null,
+            httpClient,
+            true,
+            requestBuilder,
+            null);
+
+    DropChannelRequest request =
+        DropChannelRequest.builder("channel")
+            .setDBName("db")
+            .setTableName("table")
+            .setSchemaName("schema")
+            .build();
+    client.dropChannel(request);
+    Mockito.verify(requestBuilder)
+        .generateStreamingIngestPostRequest(
+            ArgumentMatchers.contains("channel"),
+            ArgumentMatchers.refEq(DROP_CHANNEL_ENDPOINT),
+            ArgumentMatchers.refEq("drop channel"));
+  }
+
   @Test
   public void testGetChannelsStatusWithRequestError() throws Exception {
     ChannelsStatusResponse response = new ChannelsStatusResponse();
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index f3ee8dfe6..03c647b4b 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -2,6 +2,7 @@
 
 import static net.snowflake.ingest.utils.Constants.BLOB_NO_HEADER;
 import static net.snowflake.ingest.utils.Constants.COMPRESS_BLOB_TWICE;
+import static net.snowflake.ingest.utils.Constants.DROP_CHANNEL_ENDPOINT;
 import static net.snowflake.ingest.utils.Constants.REGISTER_BLOB_ENDPOINT;
 import static net.snowflake.ingest.utils.ParameterProvider.BDEC_PARQUET_COMPRESSION_ALGORITHM;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -202,6 +203,40 @@ public void testSimpleIngest() throws Exception {
     Assert.fail("Row sequencer not updated before timeout");
   }
 
+  @Test
+  public void testDropChannel() throws Exception {
+    SnowflakeURL url = new SnowflakeURL(TestUtils.getAccountURL());
+    RequestBuilder requestBuilder =
+        Mockito.spy(
+            new RequestBuilder(
+                url,
+                TestUtils.getUser(),
+                TestUtils.getKeyPair(),
+                HttpUtil.getHttpClient(url.getAccount()),
+                "testrequestbuilder"));
+    client.injectRequestBuilder(requestBuilder);
+
+    OpenChannelRequest request1 =
+        OpenChannelRequest.builder("CHANNEL")
+            .setDBName(testDb)
+            .setSchemaName(TEST_SCHEMA)
+            .setTableName(TEST_TABLE)
+            .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+            .build();
+
+    // Open a streaming ingest channel from the given client
+    SnowflakeStreamingIngestChannel channel1 = client.openChannel(request1);
+    // Close the channel after insertion
+    channel1.close(true).get();
+
+    // verify expected request sent to server
+    Mockito.verify(requestBuilder)
+        .generateStreamingIngestPostRequest(
+            ArgumentMatchers.contains("client_sequencer"),
+            ArgumentMatchers.refEq(DROP_CHANNEL_ENDPOINT),
+            ArgumentMatchers.refEq("drop channel"));
+  }
+
   @Test
   public void testParameterOverrides() throws Exception {
     Map<String, Object> parameterMap = new HashMap<>();

From b2981e81d942727bfae3fbbbbe7d193ffcbb4b6f Mon Sep 17 00:00:00 2001
From: Purujit Saha <purujit.saha@snowflake.com>
Date: Thu, 22 Feb 2024 12:32:07 -0800
Subject: [PATCH 326/356] =?UTF-8?q?Add=20more=20detailed=20javadoc=20for?=
 =?UTF-8?q?=20the=20drop=20related=20api=20to=20warn=20about=20the=20?=
 =?UTF-8?q?=E2=80=A6=20(#681)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add more detailed javadoc for the drop related api to warn about the pitfalls

* Minor update
---
 .../streaming/SnowflakeStreamingIngestChannel.java   |  6 ++++++
 .../streaming/SnowflakeStreamingIngestClient.java    | 12 +++++++++---
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
index b8687358e..7ffbf2549 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestChannel.java
@@ -82,6 +82,12 @@ public interface SnowflakeStreamingIngestChannel {
   /**
    * Close the channel, this function will make sure all the data in this channel is committed
    *
+   * <p>Note that this call with drop=true will delete <a
+   * href=https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview#offset-tokens>Offset
+   * Token</a> and other state from Snowflake servers unless the channel has already been opened in
+   * another client. So only use it if you are completely done ingesting data for this channel. If
+   * you open a channel with the same name in the future, it will behave like a new channel.
+   *
    * @param drop if true, the channel will be dropped after all data is successfully committed.
    * @return a completable future which will be completed when the channel is closed
    */
diff --git a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
index 8b65b5cae..2de8a45c2 100644
--- a/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
+++ b/src/main/java/net/snowflake/ingest/streaming/SnowflakeStreamingIngestClient.java
@@ -28,9 +28,15 @@ public interface SnowflakeStreamingIngestClient extends AutoCloseable {
   /**
    * Drop the specified channel on the server using a {@link DropChannelRequest}
    *
-   * <p>Note that this call will blindly drop the latest version of the channel and any pending data
-   * will be lost. Also see {@link SnowflakeStreamingIngestChannel#close(boolean)} to drop channels
-   * on close. That approach will drop the local version of the channel and if the channel has been
+   * <p>Note that this call will <em>blindly</em> drop the latest version of the channel and any
+   * pending data will be lost. It will also delete <a
+   * href=https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-overview#offset-tokens>Offset
+   * Token</a> and other state from Snowflake servers. So only use it if you are completely done
+   * ingesting data for this channel. If you open a channel with the same name in the future, it
+   * will behave like a new channel.
+   *
+   * <p>Also see {@link SnowflakeStreamingIngestChannel#close(boolean)} to drop channels on close.
+   * That approach will drop the local version of the channel and if the channel has been
    * concurrently reopened by another client, that version of the channel won't be affected.
    *
    * @param request the drop channel request

From 66b384c66bdd6ba77e9dd664ad54b1f60f4c9475 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Mon, 26 Feb 2024 21:06:05 -0800
Subject: [PATCH 327/356] SNOW-979849: Support user to pass a user defined
 offset token verification logic as part of channel creation (#680)

With this PR, user could pass in a user defined offset token verification function that is used to verify offset behaviors, if the verification failed, we will log an error and report it to SF
---
 .../ingest/connection/TelemetryService.java   | 32 +++++++++-
 .../OffsetTokenVerificationFunction.java      | 62 +++++++++++++++++++
 .../ingest/streaming/OpenChannelRequest.java  | 15 +++++
 .../streaming/internal/AbstractRowBuffer.java | 56 ++++++++++++++++-
 .../streaming/internal/ParquetRowBuffer.java  | 10 ++-
 ...nowflakeStreamingIngestChannelFactory.java | 12 +++-
 ...owflakeStreamingIngestChannelInternal.java | 11 +++-
 ...nowflakeStreamingIngestClientInternal.java |  1 +
 .../connection/TelemetryServiceTest.java      | 14 +++++
 .../streaming/internal/FlushServiceTest.java  |  3 +-
 .../streaming/internal/RowBufferTest.java     |  4 +-
 .../SnowflakeStreamingIngestClientTest.java   | 24 ++++---
 .../streaming/internal/StreamingIngestIT.java | 43 +++++++++++++
 13 files changed, 266 insertions(+), 21 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/streaming/OffsetTokenVerificationFunction.java

diff --git a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java
index ee1e1353d..15e2a0c5a 100644
--- a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java
+++ b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java
@@ -10,6 +10,9 @@
 import com.codahale.metrics.Meter;
 import com.codahale.metrics.Snapshot;
 import com.codahale.metrics.Timer;
+import com.google.common.util.concurrent.RateLimiter;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
 import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper;
@@ -28,7 +31,8 @@ enum TelemetryType {
     STREAMING_INGEST_LATENCY_IN_SEC("streaming_ingest_latency_in_ms"),
     STREAMING_INGEST_CLIENT_FAILURE("streaming_ingest_client_failure"),
     STREAMING_INGEST_THROUGHPUT_BYTES_PER_SEC("streaming_ingest_throughput_bytes_per_sec"),
-    STREAMING_INGEST_CPU_MEMORY_USAGE("streaming_ingest_cpu_memory_usage");
+    STREAMING_INGEST_CPU_MEMORY_USAGE("streaming_ingest_cpu_memory_usage"),
+    STREAMING_INGEST_BATCH_OFFSET_MISMATCH("streaming_ingest_batch_offset_mismatch");
 
     private final String name;
 
@@ -55,6 +59,7 @@ public String toString() {
   private static final String PERCENTILE99TH = "99thPercentile";
   private final TelemetryClient telemetry;
   private final String clientName;
+  private final Map<String, RateLimiter> rateLimitersMap;
 
   /**
    * Default constructor
@@ -66,6 +71,7 @@ public String toString() {
   TelemetryService(CloseableHttpClient httpClient, String clientName, String url) {
     this.clientName = clientName;
     this.telemetry = (TelemetryClient) TelemetryClient.createSessionlessTelemetry(httpClient, url);
+    this.rateLimitersMap = new HashMap<>();
   }
 
   /** Flush the telemetry buffer and close the telemetry service */
@@ -122,6 +128,30 @@ public void reportCpuMemoryUsage(Histogram cpuUsage) {
     }
   }
 
+  /** Report the offset token mismatch in a batch */
+  public void reportBatchOffsetMismatch(
+      String channelName,
+      String prevBatchEndOffset,
+      String startOffset,
+      String endOffset,
+      long rowCount) {
+    // Add a rate limiter to report the mismatch at most once every second per channel
+    RateLimiter rateLimiter =
+        rateLimitersMap.computeIfAbsent(channelName, v -> RateLimiter.create(1.0));
+    if (rateLimiter.tryAcquire()) {
+      ObjectNode msg = MAPPER.createObjectNode();
+      msg.put("channel_name", channelName);
+      msg.put("prev_batch_end_offset", prevBatchEndOffset);
+      msg.put("start_offset", startOffset);
+      msg.put("end_offset", endOffset);
+      msg.put("row_count", rowCount);
+      send(TelemetryType.STREAMING_INGEST_BATCH_OFFSET_MISMATCH, msg);
+    } else {
+      logger.logDebug(
+          "Rate limit exceeded on reportBatchOffsetMismatch, skipping report it to SF.");
+    }
+  }
+
   /** Send log to Snowflake asynchronously through JDBC client telemetry */
   void send(TelemetryType type, ObjectNode msg) {
     try {
diff --git a/src/main/java/net/snowflake/ingest/streaming/OffsetTokenVerificationFunction.java b/src/main/java/net/snowflake/ingest/streaming/OffsetTokenVerificationFunction.java
new file mode 100644
index 000000000..847e56501
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/streaming/OffsetTokenVerificationFunction.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2024 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.streaming;
+
+/**
+ * Interface to provide a custom offset verification logic. If specified, verification failures will
+ * be logged as warnings and reported to Snowflake. This interface could be used when there are
+ * certain assumption about the offset token behavior. Please reach out to Snowflake if you notice
+ * any unexpected behaviors.
+ *
+ * <p>Below is an example that verifies that all offset tokens are monotonically increasing numbers:
+ *
+ * <pre>
+ *     <code>
+ *      private static final OffsetTokenVerificationFunction offsetTokenVerificationFunction =
+ *       (prevBatchEndOffset, curBatchStartOffset, curBatchEndOffset, rowCount) -> {
+ *         boolean isMatch = true;
+ *
+ *         if (curBatchStartOffset != null) {
+ *           try {
+ *             long curStart = Long.parseLong(curBatchStartOffset);
+ *             long curEnd = Long.parseLong(curBatchEndOffset);
+ *
+ *             // We verify that the end_offset - start_offset + 1 = row_count
+ *             if (curEnd - curStart + 1 != rowCount) {
+ *               isMatch = false;
+ *             }
+ *
+ *             // We verify that start_offset_of_current_batch = end_offset_of_previous_batch+1
+ *             if (prevBatchEndOffset != null) {
+ *               long prevEnd = Long.parseLong(prevBatchEndOffset);
+ *               if (curStart != prevEnd + 1) {
+ *                 isMatch = false;
+ *               }
+ *             }
+ *           } catch (NumberFormatException ignored) {
+ *             // Do nothing since we can't compare the offset, or report a mismatch if number is expected
+ *           }
+ *         }
+ *
+ *         return isMatch;
+ *       };
+ *     </code>
+ * </pre>
+ */
+public interface OffsetTokenVerificationFunction {
+  /**
+   * @param prevBatchEndOffset end offset token of the previous batch
+   * @param curBatchStartOffset start offset token of the current batch
+   * @param curBatchEndOffset end offset token of the current batch
+   * @param rowCount number of rows in the current batch
+   * @return a boolean indicates whether the verification passed or not, if not, we will log a
+   *     warning and report it to SF
+   */
+  boolean verify(
+      String prevBatchEndOffset,
+      String curBatchStartOffset,
+      String curBatchEndOffset,
+      long rowCount);
+}
diff --git a/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java b/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java
index 0cf16af2e..cc8782dbd 100644
--- a/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java
+++ b/src/main/java/net/snowflake/ingest/streaming/OpenChannelRequest.java
@@ -43,6 +43,8 @@ public enum OnErrorOption {
   private final String offsetToken;
   private final boolean isOffsetTokenProvided;
 
+  private final OffsetTokenVerificationFunction offsetTokenVerificationFunction;
+
   public static OpenChannelRequestBuilder builder(String channelName) {
     return new OpenChannelRequestBuilder(channelName);
   }
@@ -59,6 +61,8 @@ public static class OpenChannelRequestBuilder {
     private String offsetToken;
     private boolean isOffsetTokenProvided = false;
 
+    private OffsetTokenVerificationFunction offsetTokenVerificationFunction;
+
     public OpenChannelRequestBuilder(String channelName) {
       this.channelName = channelName;
       this.defaultTimezone = DEFAULT_DEFAULT_TIMEZONE;
@@ -95,6 +99,12 @@ public OpenChannelRequestBuilder setOffsetToken(String offsetToken) {
       return this;
     }
 
+    public OpenChannelRequestBuilder setOffsetTokenVerificationFunction(
+        OffsetTokenVerificationFunction function) {
+      this.offsetTokenVerificationFunction = function;
+      return this;
+    }
+
     public OpenChannelRequest build() {
       return new OpenChannelRequest(this);
     }
@@ -116,6 +126,7 @@ private OpenChannelRequest(OpenChannelRequestBuilder builder) {
     this.defaultTimezone = builder.defaultTimezone;
     this.offsetToken = builder.offsetToken;
     this.isOffsetTokenProvided = builder.isOffsetTokenProvided;
+    this.offsetTokenVerificationFunction = builder.offsetTokenVerificationFunction;
   }
 
   public String getDBName() {
@@ -153,4 +164,8 @@ public String getOffsetToken() {
   public boolean isOffsetTokenProvided() {
     return this.isOffsetTokenProvided;
   }
+
+  public OffsetTokenVerificationFunction getOffsetTokenVerificationFunction() {
+    return this.offsetTokenVerificationFunction;
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
index 6f7edd921..4a13f5adb 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/AbstractRowBuffer.java
@@ -17,7 +17,9 @@
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
+import net.snowflake.ingest.connection.TelemetryService;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
+import net.snowflake.ingest.streaming.OffsetTokenVerificationFunction;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
@@ -173,6 +175,8 @@ public InsertValidationResponse insertRows(
         rowIndex++;
       }
       checkBatchSizeRecommendedMaximum(rowsSizeInBytes);
+      checkOffsetMismatch(
+          rowBuffer.channelState.getEndOffsetToken(), startOffsetToken, endOffsetToken, rowIndex);
       rowBuffer.channelState.updateOffsetToken(startOffsetToken, endOffsetToken, prevRowCount);
       rowBuffer.bufferSize += rowsSizeInBytes;
       rowBuffer.rowSizeMetric.accept(rowsSizeInBytes);
@@ -213,6 +217,11 @@ public InsertValidationResponse insertRows(
               rowBuffer.statsMap.put(
                   colName,
                   RowBufferStats.getCombinedStats(stats, rowBuffer.tempStatsMap.get(colName))));
+      checkOffsetMismatch(
+          rowBuffer.channelState.getEndOffsetToken(),
+          startOffsetToken,
+          endOffsetToken,
+          tempRowCount);
       rowBuffer.channelState.updateOffsetToken(
           startOffsetToken, endOffsetToken, rowBuffer.bufferedRowCount);
       rowBuffer.bufferedRowCount += tempRowCount;
@@ -267,6 +276,8 @@ public InsertValidationResponse insertRows(
                 rowBuffer.statsMap.put(
                     colName,
                     RowBufferStats.getCombinedStats(stats, rowBuffer.tempStatsMap.get(colName))));
+        checkOffsetMismatch(
+            rowBuffer.channelState.getEndOffsetToken(), startOffsetToken, endOffsetToken, rowIndex);
         rowBuffer.channelState.updateOffsetToken(
             startOffsetToken, endOffsetToken, rowBuffer.bufferedRowCount);
         rowBuffer.bufferedRowCount += tempRowCount;
@@ -312,13 +323,21 @@ public InsertValidationResponse insertRows(
   // Buffer parameters that are set at the owning client level
   final ClientBufferParameters clientBufferParameters;
 
+  // Function used to verify offset token logic
+  final OffsetTokenVerificationFunction offsetTokenVerificationFunction;
+
+  // Telemetry service use to report telemetry to SF
+  final TelemetryService telemetryService;
+
   AbstractRowBuffer(
       OpenChannelRequest.OnErrorOption onErrorOption,
       ZoneId defaultTimezone,
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelRuntimeState,
-      ClientBufferParameters clientBufferParameters) {
+      ClientBufferParameters clientBufferParameters,
+      OffsetTokenVerificationFunction offsetTokenVerificationFunction,
+      TelemetryService telemetryService) {
     this.onErrorOption = onErrorOption;
     this.defaultTimezone = defaultTimezone;
     this.rowSizeMetric = rowSizeMetric;
@@ -329,6 +348,8 @@ public InsertValidationResponse insertRows(
     this.bufferedRowCount = 0;
     this.bufferSize = 0F;
     this.clientBufferParameters = clientBufferParameters;
+    this.offsetTokenVerificationFunction = offsetTokenVerificationFunction;
+    this.telemetryService = telemetryService;
 
     // Initialize empty stats
     this.statsMap = new HashMap<>();
@@ -630,7 +651,9 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelRuntimeState,
-      ClientBufferParameters clientBufferParameters) {
+      ClientBufferParameters clientBufferParameters,
+      OffsetTokenVerificationFunction offsetTokenVerificationFunction,
+      TelemetryService telemetryService) {
     switch (bdecVersion) {
       case THREE:
         //noinspection unchecked
@@ -641,7 +664,9 @@ static <T> AbstractRowBuffer<T> createRowBuffer(
                 fullyQualifiedChannelName,
                 rowSizeMetric,
                 channelRuntimeState,
-                clientBufferParameters);
+                clientBufferParameters,
+                offsetTokenVerificationFunction,
+                telemetryService);
       default:
         throw new SFException(
             ErrorCode.INTERNAL_ERROR, "Unsupported BDEC format version: " + bdecVersion);
@@ -669,6 +694,31 @@ private void checkBatchSizeRecommendedMaximum(float batchSizeInBytes) {
     }
   }
 
+  /**
+   * We verify the offset token based on the provided verification logic and report to SF if there
+   * is a mismatch.
+   */
+  private void checkOffsetMismatch(
+      String prevEndOffset, String curStartOffset, String curEndOffset, int rowCount) {
+    if (offsetTokenVerificationFunction != null
+        && !offsetTokenVerificationFunction.verify(
+            prevEndOffset, curStartOffset, curEndOffset, rowCount)) {
+      logger.logWarn(
+          "The channel {} might have an offset token mismatch based on the provided offset token"
+              + " verification logic, preEndOffset={}, curStartOffset={}, curEndOffset={},"
+              + " rowCount={}.",
+          channelFullyQualifiedName,
+          prevEndOffset,
+          curStartOffset,
+          curEndOffset,
+          rowCount);
+      if (telemetryService != null) {
+        telemetryService.reportBatchOffsetMismatch(
+            channelFullyQualifiedName, prevEndOffset, curStartOffset, curEndOffset, rowCount);
+      }
+    }
+  }
+
   /** Create the ingestion strategy based on the channel on_error option */
   IngestionStrategy<T> createIngestionStrategy(OpenChannelRequest.OnErrorOption onErrorOption) {
     switch (onErrorOption) {
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
index 3cf7198e7..17aaa9136 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java
@@ -19,6 +19,8 @@
 import java.util.Set;
 import java.util.function.Consumer;
 import net.snowflake.client.jdbc.internal.google.common.collect.Sets;
+import net.snowflake.ingest.connection.TelemetryService;
+import net.snowflake.ingest.streaming.OffsetTokenVerificationFunction;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.utils.Constants;
 import net.snowflake.ingest.utils.ErrorCode;
@@ -58,14 +60,18 @@ public class ParquetRowBuffer extends AbstractRowBuffer<ParquetChunkData> {
       String fullyQualifiedChannelName,
       Consumer<Float> rowSizeMetric,
       ChannelRuntimeState channelRuntimeState,
-      ClientBufferParameters clientBufferParameters) {
+      ClientBufferParameters clientBufferParameters,
+      OffsetTokenVerificationFunction offsetTokenVerificationFunction,
+      TelemetryService telemetryService) {
     super(
         onErrorOption,
         defaultTimezone,
         fullyQualifiedChannelName,
         rowSizeMetric,
         channelRuntimeState,
-        clientBufferParameters);
+        clientBufferParameters,
+        offsetTokenVerificationFunction,
+        telemetryService);
     this.fieldIndex = new HashMap<>();
     this.metadata = new HashMap<>();
     this.data = new ArrayList<>();
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java
index 3e442d43b..a56b82ed5 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelFactory.java
@@ -5,6 +5,7 @@
 package net.snowflake.ingest.streaming.internal;
 
 import java.time.ZoneId;
+import net.snowflake.ingest.streaming.OffsetTokenVerificationFunction;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.utils.Utils;
 
@@ -27,8 +28,8 @@ static class SnowflakeStreamingIngestChannelBuilder<T> {
     private String encryptionKey;
     private Long encryptionKeyId;
     private OpenChannelRequest.OnErrorOption onErrorOption;
-
     private ZoneId defaultTimezone;
+    private OffsetTokenVerificationFunction offsetTokenVerificationFunction;
 
     private SnowflakeStreamingIngestChannelBuilder(String name) {
       this.name = name;
@@ -91,6 +92,12 @@ SnowflakeStreamingIngestChannelBuilder<T> setOwningClient(
       return this;
     }
 
+    SnowflakeStreamingIngestChannelBuilder<T> setOffsetTokenVerificationFunction(
+        OffsetTokenVerificationFunction function) {
+      this.offsetTokenVerificationFunction = function;
+      return this;
+    }
+
     SnowflakeStreamingIngestChannelInternal<T> build() {
       Utils.assertStringNotNullOrEmpty("channel name", this.name);
       Utils.assertStringNotNullOrEmpty("table name", this.tableName);
@@ -116,7 +123,8 @@ SnowflakeStreamingIngestChannelInternal<T> build() {
           this.encryptionKeyId,
           this.onErrorOption,
           this.defaultTimezone,
-          this.owningClient.getParameterProvider().getBlobFormatVersion());
+          this.owningClient.getParameterProvider().getBlobFormatVersion(),
+          this.offsetTokenVerificationFunction);
     }
   }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
index 2f4a7678d..578f8e6c1 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelInternal.java
@@ -22,6 +22,7 @@
 import javax.annotation.Nullable;
 import net.snowflake.ingest.streaming.DropChannelRequest;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
+import net.snowflake.ingest.streaming.OffsetTokenVerificationFunction;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.utils.Constants;
@@ -95,7 +96,8 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
         encryptionKeyId,
         onErrorOption,
         defaultTimezone,
-        client.getParameterProvider().getBlobFormatVersion());
+        client.getParameterProvider().getBlobFormatVersion(),
+        null);
   }
 
   /** Default constructor */
@@ -112,7 +114,8 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
       Long encryptionKeyId,
       OpenChannelRequest.OnErrorOption onErrorOption,
       ZoneId defaultTimezone,
-      Constants.BdecVersion bdecVersion) {
+      Constants.BdecVersion bdecVersion,
+      OffsetTokenVerificationFunction offsetTokenVerificationFunction) {
     this.isClosed = false;
     this.owningClient = client;
     this.channelFlushContext =
@@ -127,7 +130,9 @@ class SnowflakeStreamingIngestChannelInternal<T> implements SnowflakeStreamingIn
             getFullyQualifiedName(),
             this::collectRowSize,
             channelState,
-            new ClientBufferParameters(owningClient));
+            new ClientBufferParameters(owningClient),
+            offsetTokenVerificationFunction,
+            owningClient == null ? null : owningClient.getTelemetryService());
     this.tableColumns = new HashMap<>();
     logger.logInfo(
         "Channel={} created for table={}",
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 0dde314f4..62fc265ff 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -356,6 +356,7 @@ public SnowflakeStreamingIngestChannelInternal<?> openChannel(OpenChannelRequest
               .setEncryptionKeyId(response.getEncryptionKeyId())
               .setOnErrorOption(request.getOnErrorOption())
               .setDefaultTimezone(request.getDefaultTimezone())
+              .setOffsetTokenVerificationFunction(request.getOffsetTokenVerificationFunction())
               .build();
 
       // Setup the row buffer schema
diff --git a/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java b/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java
index 962aa1a81..91a8449a7 100644
--- a/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java
@@ -73,4 +73,18 @@ public void testReportCpuMemoryUsage() {
     // Make sure there is no exception thrown
     telemetryService.reportCpuMemoryUsage(cpuHistogram);
   }
+
+  @Test
+  public void testReportBatchOffsetMismatch() {
+    CloseableHttpClient httpClient = Mockito.mock(CloseableHttpClient.class);
+
+    TelemetryService telemetryService =
+        Mockito.spy(
+            new TelemetryService(
+                httpClient, "testReportClientFailure", "snowflake.dev.local:8082"));
+    Mockito.doNothing().when(telemetryService).send(Mockito.any(), Mockito.any());
+
+    // Make sure there is no exception thrown
+    telemetryService.reportBatchOffsetMismatch("channel", "0", "1", "2", 1);
+  }
 }
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
index a25fd416e..28d1206a2 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/FlushServiceTest.java
@@ -258,7 +258,8 @@ SnowflakeStreamingIngestChannelInternal<List<List<Object>>> createChannel(
           encryptionKeyId,
           onErrorOption,
           defaultTimezone,
-          Constants.BdecVersion.THREE);
+          Constants.BdecVersion.THREE,
+          null);
     }
 
     @Override
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
index e593c3d94..5e34bc90b 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/RowBufferTest.java
@@ -126,7 +126,9 @@ private AbstractRowBuffer<?> createTestBuffer(OpenChannelRequest.OnErrorOption o
             enableParquetMemoryOptimization,
             MAX_CHUNK_SIZE_IN_BYTES_DEFAULT,
             MAX_ALLOWED_ROW_SIZE_IN_BYTES_DEFAULT,
-            Constants.BdecParquetCompression.GZIP));
+            Constants.BdecParquetCompression.GZIP),
+        null,
+        null);
   }
 
   @Test
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
index dda34d83d..1693e1520 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java
@@ -97,7 +97,8 @@ public void setup() {
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
             ZoneOffset.UTC,
-            BDEC_VERSION);
+            BDEC_VERSION,
+            null);
     channel2 =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel2",
@@ -112,7 +113,8 @@ public void setup() {
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
             ZoneOffset.UTC,
-            BDEC_VERSION);
+            BDEC_VERSION,
+            null);
     channel3 =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel3",
@@ -127,7 +129,8 @@ public void setup() {
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
             ZoneOffset.UTC,
-            BDEC_VERSION);
+            BDEC_VERSION,
+            null);
     channel4 =
         new SnowflakeStreamingIngestChannelInternal<>(
             "channel4",
@@ -142,7 +145,8 @@ public void setup() {
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
             ZoneOffset.UTC,
-            BDEC_VERSION);
+            BDEC_VERSION,
+            null);
   }
 
   @Test
@@ -358,7 +362,8 @@ public void testGetChannelsStatusWithRequest() throws Exception {
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
             ZoneOffset.UTC,
-            BDEC_VERSION);
+            BDEC_VERSION,
+            null);
 
     ChannelsStatusRequest.ChannelStatusRequestDTO dto =
         new ChannelsStatusRequest.ChannelStatusRequestDTO(channel);
@@ -462,7 +467,8 @@ public void testGetChannelsStatusWithRequestError() throws Exception {
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
             ZoneOffset.UTC,
-            BDEC_VERSION);
+            BDEC_VERSION,
+            null);
 
     try {
       client.getChannelsStatus(Collections.singletonList(channel));
@@ -503,7 +509,8 @@ public void testRegisterBlobRequestCreationSuccess() throws Exception {
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
             ZoneOffset.UTC,
-            BDEC_VERSION);
+            BDEC_VERSION,
+            null);
 
     ChannelMetadata channelMetadata =
         ChannelMetadata.builder()
@@ -1419,7 +1426,8 @@ public void testGetLatestCommittedOffsetTokens() throws Exception {
             1234L,
             OpenChannelRequest.OnErrorOption.CONTINUE,
             ZoneOffset.UTC,
-            BDEC_VERSION);
+            BDEC_VERSION,
+            null);
 
     ChannelsStatusRequest.ChannelStatusRequestDTO dto =
         new ChannelsStatusRequest.ChannelStatusRequestDTO(channel);
diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
index 03c647b4b..17f4d31fe 100644
--- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
+++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestIT.java
@@ -34,6 +34,7 @@
 import net.snowflake.ingest.TestUtils;
 import net.snowflake.ingest.connection.RequestBuilder;
 import net.snowflake.ingest.streaming.InsertValidationResponse;
+import net.snowflake.ingest.streaming.OffsetTokenVerificationFunction;
 import net.snowflake.ingest.streaming.OpenChannelRequest;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel;
 import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClientFactory;
@@ -78,6 +79,35 @@ public static Object[] compressionAlgorithms() {
 
   @Parameter public String compressionAlgorithm;
 
+  private static final OffsetTokenVerificationFunction offsetTokenVerificationFunction =
+      (prevBatchEndOffset, curBatchStartOffset, curBatchEndOffset, rowCount) -> {
+        boolean isMatch = true;
+
+        if (curBatchStartOffset != null) {
+          try {
+            long curStart = Long.parseLong(curBatchStartOffset);
+            long curEnd = Long.parseLong(curBatchEndOffset);
+
+            // We verify that the end_offset - start_offset + 1 = row_count
+            if (curEnd - curStart + 1 != rowCount) {
+              isMatch = false;
+            }
+
+            // We verify that start_offset_of_current_batch = end_offset_of_previous_batch+1
+            if (prevBatchEndOffset != null) {
+              long prevEnd = Long.parseLong(prevBatchEndOffset);
+              if (curStart != prevEnd + 1) {
+                isMatch = false;
+              }
+            }
+          } catch (NumberFormatException ignored) {
+            // Do nothing since we can't compare the offset
+          }
+        }
+
+        return isMatch;
+      };
+
   @Before
   public void beforeAll() throws Exception {
     testDb = TEST_DB_PREFIX + "_" + UUID.randomUUID().toString().substring(0, 4);
@@ -1012,6 +1042,7 @@ public void testInsertRowsWithValidStartOffsetToken() throws Exception {
             .setSchemaName(TEST_SCHEMA)
             .setTableName(insertRowsWithValidStartOffsetToken)
             .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+            .setOffsetTokenVerificationFunction(offsetTokenVerificationFunction)
             .build();
 
     // Open a streaming ingest channel from the given client
@@ -1068,6 +1099,7 @@ public void testInsertRowsWithInvalidStartOffsetToken() throws Exception {
             .setSchemaName(TEST_SCHEMA)
             .setTableName(insertRowsWithInvalidStartOffsetToken)
             .setOnErrorOption(OpenChannelRequest.OnErrorOption.CONTINUE)
+            .setOffsetTokenVerificationFunction(offsetTokenVerificationFunction)
             .build();
 
     // Open a streaming ingest channel from the given client
@@ -1107,6 +1139,17 @@ public void testInsertRowsWithInvalidStartOffsetToken() throws Exception {
     Assert.fail("Row sequencer not updated before timeout");
   }
 
+  @Test
+  public void testOffsetTokenVerificationFunction() {
+    Assert.assertTrue(offsetTokenVerificationFunction.verify("1", "2", "4", 3));
+    Assert.assertTrue(offsetTokenVerificationFunction.verify(null, "2", "4", 3));
+    Assert.assertTrue(offsetTokenVerificationFunction.verify("1", "2", null, 3));
+    Assert.assertTrue(offsetTokenVerificationFunction.verify("a", "2", "4", 3));
+    Assert.assertFalse(offsetTokenVerificationFunction.verify("1", "3", "4", 3));
+    Assert.assertFalse(offsetTokenVerificationFunction.verify("2", "1", "4", 3));
+    Assert.assertFalse(offsetTokenVerificationFunction.verify("1", "2", "4", 2));
+  }
+
   @Test
   public void testChannelClose() throws Exception {
     OpenChannelRequest request1 =

From a61fd252c75d3b0c8541a0f5b764ba418200d2e9 Mon Sep 17 00:00:00 2001
From: Toby Zhang <toby.zhang@snowflake.com>
Date: Tue, 27 Feb 2024 09:16:19 -0800
Subject: [PATCH 328/356] V2.1.0 Release (#682)

Let's do a minor version upgrade since:

- We upgrade the JDBC version
- We have a change which might introduce some behavior changes
---
 e2e-jar-test/pom.xml                          |  2 +-
 pom.xml                                       | 26 +++++++++----------
 .../ingest/connection/RequestBuilder.java     |  2 +-
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/e2e-jar-test/pom.xml b/e2e-jar-test/pom.xml
index 1012be2df..27e92432b 100644
--- a/e2e-jar-test/pom.xml
+++ b/e2e-jar-test/pom.xml
@@ -27,7 +27,7 @@
             <dependency>
                 <groupId>net.snowflake</groupId>
                 <artifactId>snowflake-ingest-sdk</artifactId>
-                <version>2.0.5</version>
+                <version>2.1.0</version>
             </dependency>
 
             <dependency>
diff --git a/pom.xml b/pom.xml
index 8992b1264..7472adcae 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
   <!-- Arifact name and version information -->
   <groupId>net.snowflake</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.0.5</version>
+  <version>2.1.0</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
@@ -786,9 +786,9 @@
         <configuration>
           <errorRemedy>failFast</errorRemedy>
           <!--
-                               The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please
-                              check your dependencies", verify the conditions of the license and add the reference to it here.
-                            -->
+                                         The list of allowed licenses. If you see the build failing due to "There are some forbidden licenses used, please
+                                        check your dependencies", verify the conditions of the license and add the reference to it here.
+                                      -->
           <includedLicenses>
             <includedLicense>Apache License 2.0</includedLicense>
             <includedLicense>BSD 2-Clause License</includedLicense>
@@ -906,9 +906,9 @@
       <build>
         <plugins>
           <!--
-                                        Copy all project dependencies to target/dependency-jars. License processing Python script will look here for
-                                        license files of SDK dependencies.
-                                        -->
+                                                  Copy all project dependencies to target/dependency-jars. License processing Python script will look here for
+                                                  license files of SDK dependencies.
+                                                  -->
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-dependency-plugin</artifactId>
@@ -930,9 +930,9 @@
           </plugin>
 
           <!--
-                                        Compile the list of SDK dependencies in 'compile' and 'runtime' scopes.
-                                        This list is an entry point for the license processing python script.
-                                        -->
+                                                  Compile the list of SDK dependencies in 'compile' and 'runtime' scopes.
+                                                  This list is an entry point for the license processing python script.
+                                                  -->
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-dependency-plugin</artifactId>
@@ -1112,9 +1112,9 @@
             </executions>
           </plugin>
           <!--
-                                        Plugin executes license processing Python script, which copies third party license files into the directory
-                                        target/generated-licenses-info/META-INF/third-party-licenses, which is then included in the shaded JAR.
-                                        -->
+                                                  Plugin executes license processing Python script, which copies third party license files into the directory
+                                                  target/generated-licenses-info/META-INF/third-party-licenses, which is then included in the shaded JAR.
+                                                  -->
           <plugin>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>exec-maven-plugin</artifactId>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 26911501b..e84812564 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.0.5";
+  public static final String DEFAULT_VERSION = "2.1.0";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From fde1aa629b1a1c50e2fb485bfd08427ed1704e34 Mon Sep 17 00:00:00 2001
From: Nirut Gupta <nirgupta@confluent.io>
Date: Wed, 5 Feb 2025 22:05:18 +0530
Subject: [PATCH 329/356] close httpClient when closing simpleIngestManager

---
 src/main/java/net/snowflake/ingest/SimpleIngestManager.java | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
index 17caa2b9b..b5462544d 100644
--- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
+++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
@@ -608,6 +608,11 @@ public HistoryRangeResponse getHistoryRange(
   @Override
   public void close() {
     builder.closeResources();
+    try {
+      httpClient.close();
+    } catch (IOException e) {
+      LOGGER.error("Error closing http client", e);
+    }
     HttpUtil.shutdownHttpConnectionManagerDaemonThread();
   }
 

From fcdeafba3a25cd71b3cd1e5315915a795fd57322 Mon Sep 17 00:00:00 2001
From: Nirut Gupta <nirgupta@confluent.io>
Date: Wed, 5 Feb 2025 22:16:18 +0530
Subject: [PATCH 330/356] add semaphore files

---
 .semaphore/semaphore.yml | 11 +++++------
 service.yml              | 17 +++--------------
 2 files changed, 8 insertions(+), 20 deletions(-)

diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
index ee1d13ad0..341161217 100644
--- a/.semaphore/semaphore.yml
+++ b/.semaphore/semaphore.yml
@@ -7,7 +7,7 @@ version: v1.0
 name: build-test-release
 agent:
   machine:
-    type: s1-prod-ubuntu24-04-amd64-1
+    type: s1-prod-ubuntu20-04-amd64-1
 
 fail_fast:
   cancel:
@@ -24,7 +24,7 @@ global_job_config:
   prologue:
     commands:
       - checkout
-      - sem-version java 8
+      - . set-cp-java-version
       - . cache-maven restore
 
 blocks:
@@ -32,14 +32,13 @@ blocks:
     dependencies: []
     run:
       # don't run the tests on non-functional changes...
-      when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/'], default_branch: 'master'})"
+      when: "change_in('/', {exclude: ['/.deployed-versions/', '.github/']})"
     task:
       jobs:
         - name: Test
           commands:
             - . sem-pint -c
             - mvn -U -Dmaven.wagon.http.retryHandler.count=10 --batch-mode --no-transfer-progress clean verify install dependency:analyze validate
-            - cve-scan
             - . cache-maven store
       epilogue:
         always:
@@ -63,7 +62,7 @@ after_pipeline:
   task:
     agent:
       machine:
-        type: s1-prod-ubuntu24-04-arm64-0
+        type: s1-prod-ubuntu20-04-arm64-0
     jobs:
       - name: Metrics
         commands:
@@ -76,4 +75,4 @@ after_pipeline:
           - checkout
           - sem-version java 11
           - artifact pull workflow target
-          - emit-sonarqube-data --run_only_sonar_scan
+          - emit-sonarqube-data --run_only_sonar_scan
\ No newline at end of file
diff --git a/service.yml b/service.yml
index dfb3dd22c..87348723d 100644
--- a/service.yml
+++ b/service.yml
@@ -1,21 +1,10 @@
 name: snowflake-ingest-java
-lang: java
-lang_version: 8
+lang: unknown
+lang_version: unknown
 codeowners:
   enable: true
 semaphore:
   enable: true
   pipeline_type: cp
-  cve_scan: true
-  branches:
-    - master
-    - main
-    - /^\d+\.\d+\.x$/
-    - /v\d+\.\d+\.\d+\-hotfix\-x/
-    - /^gh-readonly-queue.*/
 git:
-  enable: true
-code_artifact:
-  enable: true
-  package_paths:
-    - maven-snapshots/maven/io.confluent/snowflake-ingest-sdk
\ No newline at end of file
+  enable: true
\ No newline at end of file

From 10f825d162f20eeb15ce9ee6ad247b15e69c199d Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Thu, 6 Feb 2025 17:00:44 +0530
Subject: [PATCH 331/356] Http client leak hotfix (#14)

---
 .semaphore/semaphore.yml                      |  2 -
 README.md                                     |  2 +-
 e2e-jar-test/pom.xml                          |  2 +-
 pom.xml                                       | 89 +++++++------------
 service.yml                                   | 10 ++-
 .../ingest/connection/RequestBuilder.java     |  2 +-
 6 files changed, 41 insertions(+), 66 deletions(-)

diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
index 341161217..da45b7663 100644
--- a/.semaphore/semaphore.yml
+++ b/.semaphore/semaphore.yml
@@ -49,8 +49,6 @@ blocks:
 
   - name: Release
     dependencies: ["Test"]
-    run:
-      when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.x'"
     task:
       jobs:
         - name: Release
diff --git a/README.md b/README.md
index e0cd0d8b1..92015a17d 100644
--- a/README.md
+++ b/README.md
@@ -119,7 +119,7 @@ mvn install
 ```
 
 If you would just like to build the jar in the source directory, you can
-run
+run 
 
 ``` {.bash}
 mvn package
diff --git a/e2e-jar-test/pom.xml b/e2e-jar-test/pom.xml
index 27e92432b..8e7c72c51 100644
--- a/e2e-jar-test/pom.xml
+++ b/e2e-jar-test/pom.xml
@@ -27,7 +27,7 @@
             <dependency>
                 <groupId>net.snowflake</groupId>
                 <artifactId>snowflake-ingest-sdk</artifactId>
-                <version>2.1.0</version>
+                <version>2.1.0-hotfix1</version>
             </dependency>
 
             <dependency>
diff --git a/pom.xml b/pom.xml
index 7472adcae..0d9ffa59d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,37 +1,14 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
   <!-- Arifact name and version information -->
-  <groupId>net.snowflake</groupId>
+  <groupId>io.confluent</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.1.0</version>
+  <version>2.1.0-hotfix1</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
-  <url>https://www.snowflake.net/</url>
-
-  <licenses>
-    <license>
-      <name>The Apache Software License, Version 2.0</name>
-      <url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
-    </license>
-  </licenses>
-
-  <developers>
-    <developer>
-      <name>Snowflake Support Team</name>
-      <email>snowflake-java@snowflake.net</email>
-      <organization>Snowflake Computing</organization>
-      <organizationUrl>https://www.snowflake.net</organizationUrl>
-    </developer>
-  </developers>
-
-  <scm>
-    <connection>scm:git:git://github.com/snowflakedb/snowflake-ingest-java</connection>
-    <url>https://github.com/snowflakedb/snowflake-ingest-java/tree/master</url>
-  </scm>
 
   <!-- Set our Language Level to Java 8 -->
   <properties>
@@ -632,32 +609,6 @@
       </plugins>
     </pluginManagement>
     <plugins>
-      <plugin>
-        <groupId>com.github.ekryd.sortpom</groupId>
-        <artifactId>sortpom-maven-plugin</artifactId>
-        <version>3.0.1</version>
-        <configuration>
-          <createBackupFile>false</createBackupFile>
-          <expandEmptyElements>false</expandEmptyElements>
-          <indentSchemaLocation>true</indentSchemaLocation>
-          <sortDependencies>scope,groupId,artifactId</sortDependencies>
-          <sortDependencyExclusions>groupId,artifactId</sortDependencyExclusions>
-          <sortExecutions>true</sortExecutions>
-          <sortModules>true</sortModules>
-          <sortPlugins>groupId,artifactId</sortPlugins>
-          <sortProperties>true</sortProperties>
-          <verifyFail>stop</verifyFail>
-          <verifyFailOn>strict</verifyFailOn>
-        </configuration>
-        <executions>
-          <execution>
-            <goals>
-              <goal>verify</goal>
-            </goals>
-            <phase>validate</phase>
-          </execution>
-        </executions>
-      </plugin>
       <plugin>
         <artifactId>maven-assembly-plugin</artifactId>
         <configuration>
@@ -717,10 +668,10 @@
                   <findAllDuplicates>true</findAllDuplicates>
                   <ignoreWhenIdentical>true</ignoreWhenIdentical>
                 </banDuplicateClasses>
-                <banDuplicatePomDependencyVersions/>
-                <bannedDependencies/>
-                <dependencyConvergence/>
-                <requireUpperBoundDeps/>
+                <banDuplicatePomDependencyVersions></banDuplicatePomDependencyVersions>
+                <bannedDependencies></bannedDependencies>
+                <dependencyConvergence></dependencyConvergence>
+                <requireUpperBoundDeps></requireUpperBoundDeps>
               </rules>
             </configuration>
           </execution>
@@ -825,6 +776,28 @@
       </plugin>
     </plugins>
   </build>
+  <url>https://www.snowflake.net/</url>
+
+  <licenses>
+    <license>
+      <name>The Apache Software License, Version 2.0</name>
+      <url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
+    </license>
+  </licenses>
+
+  <developers>
+    <developer>
+      <name>Snowflake Support Team</name>
+      <email>snowflake-java@snowflake.net</email>
+      <organization>Snowflake Computing</organization>
+      <organizationUrl>https://www.snowflake.net</organizationUrl>
+    </developer>
+  </developers>
+
+  <scm>
+    <connection>scm:git:git://github.com/snowflakedb/snowflake-ingest-java</connection>
+    <url>https://github.com/snowflakedb/snowflake-ingest-java/tree/master</url>
+  </scm>
 
   <profiles>
     <profile>
@@ -1098,8 +1071,8 @@
                 </filter>
               </filters>
               <transformers>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
-                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"/>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"></transformer>
+                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"></transformer>
               </transformers>
             </configuration>
             <executions>
diff --git a/service.yml b/service.yml
index 87348723d..bcd3c2704 100644
--- a/service.yml
+++ b/service.yml
@@ -1,10 +1,14 @@
 name: snowflake-ingest-java
-lang: unknown
-lang_version: unknown
+lang: java
+lang_version: 8
 codeowners:
   enable: true
 semaphore:
   enable: true
   pipeline_type: cp
 git:
-  enable: true
\ No newline at end of file
+  enable: true
+code_artifact:
+  enable: true
+  package_paths:
+    - maven-snapshots/maven/io.confluent/snowflake-ingest-sdk
\ No newline at end of file
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index e84812564..8e12d925e 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.1.0";
+  public static final String DEFAULT_VERSION = "2.1.0-hotfix1";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 050b0fc626a0e03e7eb7c8250cd7e2ef08f255a2 Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Thu, 6 Feb 2025 17:34:34 +0530
Subject: [PATCH 332/356] add project.yml file (#15)

---
 .semaphore/project.yml | 48 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 .semaphore/project.yml

diff --git a/.semaphore/project.yml b/.semaphore/project.yml
new file mode 100644
index 000000000..5e8035546
--- /dev/null
+++ b/.semaphore/project.yml
@@ -0,0 +1,48 @@
+#
+# Copyright [2019 - 2019] Confluent Inc.
+#
+# This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common
+# template and configurations in service.yml.
+# Modifications in this file will be overwritten by generated content in the nightly run.
+# For more information, please refer to the page:
+# https://confluentinc.atlassian.net/wiki/spaces/Foundations/pages/2871296194/Add+SemaphoreCI
+apiVersion: v1alpha
+kind: Project
+metadata:
+  name: snowflake-ingest-java
+  description: ""
+spec:
+  visibility: private
+  repository:
+    url: git@github.com:confluentinc/snowflake-ingest-java.git
+    run_on:
+      - branches
+      - pull_requests
+    pipeline_file: .semaphore/semaphore.yml
+    integration_type: github_app
+    status:
+      pipeline_files:
+        - path: .semaphore/semaphore.yml
+          level: pipeline
+    whitelist:
+      branches:
+        - master
+        - main
+        - 2.1.0-hotfix
+        - /^\d+\.\d+\.x$/
+        - /^\d+\.\d+\.\d+-hotfix$/
+        - /^gh-readonly-queue.*/
+  custom_permissions: true
+  debug_permissions:
+    - empty
+    - default_branch
+    - non_default_branch
+    - pull_request
+    - forked_pull_request
+    - tag
+  attach_permissions:
+    - default_branch
+    - non_default_branch
+    - pull_request
+    - forked_pull_request
+    - tag
\ No newline at end of file

From aef3c01230ae92b0709c7b7f136a514fcb26a9bc Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Fri, 7 Feb 2025 00:54:25 +0530
Subject: [PATCH 333/356] remove extra space from README (#16)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 92015a17d..e0cd0d8b1 100644
--- a/README.md
+++ b/README.md
@@ -119,7 +119,7 @@ mvn install
 ```
 
 If you would just like to build the jar in the source directory, you can
-run 
+run
 
 ``` {.bash}
 mvn package

From e6985cb7d055dbb50322580a922aaaa3af3a6dc0 Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Fri, 7 Feb 2025 01:08:18 +0530
Subject: [PATCH 334/356] add whitelist branches (#17)

---
 .semaphore/project.yml | 1 +
 service.yml            | 9 ++++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/.semaphore/project.yml b/.semaphore/project.yml
index 5e8035546..3b212460f 100644
--- a/.semaphore/project.yml
+++ b/.semaphore/project.yml
@@ -31,6 +31,7 @@ spec:
         - 2.1.0-hotfix
         - /^\d+\.\d+\.x$/
         - /^\d+\.\d+\.\d+-hotfix$/
+        - /\d+\.\d+\.\d+-hotfix/
         - /^gh-readonly-queue.*/
   custom_permissions: true
   debug_permissions:
diff --git a/service.yml b/service.yml
index bcd3c2704..1c584a055 100644
--- a/service.yml
+++ b/service.yml
@@ -11,4 +11,11 @@ git:
 code_artifact:
   enable: true
   package_paths:
-    - maven-snapshots/maven/io.confluent/snowflake-ingest-sdk
\ No newline at end of file
+    - maven-snapshots/maven/io.confluent/snowflake-ingest-sdk
+branches:
+  - master
+  - main
+  - /^\d+\.\d+\.x$/
+  - /v\d+\.\d+\.\d+\-hotfix\-x/
+  - /\d+\.\d+\.\d+-hotfix/
+  - /^gh-readonly-queue.*/
\ No newline at end of file

From eec6da8d2aa4c7d1cf118ae1d8ce32c88933bff8 Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Fri, 7 Feb 2025 01:58:08 +0530
Subject: [PATCH 335/356] add whitespace in readme to trigger semaphore release
 job (#18)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e0cd0d8b1..92015a17d 100644
--- a/README.md
+++ b/README.md
@@ -119,7 +119,7 @@ mvn install
 ```
 
 If you would just like to build the jar in the source directory, you can
-run
+run 
 
 ``` {.bash}
 mvn package

From bbd763c64c28ad94b80afb5f8464fbc428a96900 Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Fri, 7 Feb 2025 02:07:52 +0530
Subject: [PATCH 336/356] Add github enable true and cve scan (#19)

---
 service.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/service.yml b/service.yml
index 1c584a055..c187436ad 100644
--- a/service.yml
+++ b/service.yml
@@ -6,8 +6,11 @@ codeowners:
 semaphore:
   enable: true
   pipeline_type: cp
+  cve_scan: true
 git:
   enable: true
+github:
+  enable: true
 code_artifact:
   enable: true
   package_paths:
@@ -15,6 +18,7 @@ code_artifact:
 branches:
   - master
   - main
+  - 2.1.0-hotfix
   - /^\d+\.\d+\.x$/
   - /v\d+\.\d+\.\d+\-hotfix\-x/
   - /\d+\.\d+\.\d+-hotfix/

From 456f42c79f556d64ea4d1f7f7177de117d9254b0 Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Fri, 7 Feb 2025 09:45:30 +0530
Subject: [PATCH 337/356] remove extra space from README (#20)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 92015a17d..e0cd0d8b1 100644
--- a/README.md
+++ b/README.md
@@ -119,7 +119,7 @@ mvn install
 ```
 
 If you would just like to build the jar in the source directory, you can
-run 
+run
 
 ``` {.bash}
 mvn package

From 0fc001cc7ff97d1f63505a4d755f877e51bc0070 Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Fri, 7 Feb 2025 10:00:04 +0530
Subject: [PATCH 338/356] fix indentation in project.yml (#21)

---
 .semaphore/project.yml | 44 +++++++++++++++++++++---------------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/.semaphore/project.yml b/.semaphore/project.yml
index 3b212460f..bd0bdd453 100644
--- a/.semaphore/project.yml
+++ b/.semaphore/project.yml
@@ -16,34 +16,34 @@ spec:
   repository:
     url: git@github.com:confluentinc/snowflake-ingest-java.git
     run_on:
-      - branches
-      - pull_requests
+    - branches
+    - pull_requests
     pipeline_file: .semaphore/semaphore.yml
     integration_type: github_app
     status:
       pipeline_files:
-        - path: .semaphore/semaphore.yml
-          level: pipeline
+      - path: .semaphore/semaphore.yml
+        level: pipeline
     whitelist:
       branches:
-        - master
-        - main
-        - 2.1.0-hotfix
-        - /^\d+\.\d+\.x$/
-        - /^\d+\.\d+\.\d+-hotfix$/
-        - /\d+\.\d+\.\d+-hotfix/
-        - /^gh-readonly-queue.*/
+      - master
+      - main
+      - 2.1.0-hotfix
+      - /^\d+\.\d+\.x$/
+      - /^\d+\.\d+\.\d+-hotfix$/
+      - /\d+\.\d+\.\d+-hotfix/
+      - /^gh-readonly-queue.*/
   custom_permissions: true
   debug_permissions:
-    - empty
-    - default_branch
-    - non_default_branch
-    - pull_request
-    - forked_pull_request
-    - tag
+  - empty
+  - default_branch
+  - non_default_branch
+  - pull_request
+  - forked_pull_request
+  - tag
   attach_permissions:
-    - default_branch
-    - non_default_branch
-    - pull_request
-    - forked_pull_request
-    - tag
\ No newline at end of file
+  - default_branch
+  - non_default_branch
+  - pull_request
+  - forked_pull_request
+  - tag
\ No newline at end of file

From f8369d4de06a9213fd05cdb89d9ec659e3b97195 Mon Sep 17 00:00:00 2001
From: Nirut Gupta <nirgupta@confluent.io>
Date: Fri, 7 Feb 2025 10:03:28 +0530
Subject: [PATCH 339/356] add whitelist branch v2.1.0-hotfix-x

---
 .semaphore/project.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.semaphore/project.yml b/.semaphore/project.yml
index bd0bdd453..4bfc36806 100644
--- a/.semaphore/project.yml
+++ b/.semaphore/project.yml
@@ -30,6 +30,7 @@ spec:
       - main
       - 2.1.0-hotfix
       - /^\d+\.\d+\.x$/
+      - /v\d+\.\d+\.\d+\-hotfix\-x/
       - /^\d+\.\d+\.\d+-hotfix$/
       - /\d+\.\d+\.\d+-hotfix/
       - /^gh-readonly-queue.*/

From 8c00dd1d3df5687792adf9be21d8dac5ae3d5f72 Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Fri, 7 Feb 2025 10:12:41 +0530
Subject: [PATCH 340/356] add whitespace in readme to trigger semaphore release
 job (#22)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e0cd0d8b1..92015a17d 100644
--- a/README.md
+++ b/README.md
@@ -119,7 +119,7 @@ mvn install
 ```
 
 If you would just like to build the jar in the source directory, you can
-run
+run 
 
 ``` {.bash}
 mvn package

From dd6bc948294d40c2fee9ce832c274e63f5d51a0b Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Fri, 7 Feb 2025 10:25:10 +0530
Subject: [PATCH 341/356] branches should be inside semaphore (#24)

---
 service.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/service.yml b/service.yml
index c187436ad..2b79b47ec 100644
--- a/service.yml
+++ b/service.yml
@@ -7,6 +7,14 @@ semaphore:
   enable: true
   pipeline_type: cp
   cve_scan: true
+  branches:
+    - master
+    - main
+    - 2.1.0-hotfix
+    - /^\d+\.\d+\.x$/
+    - /v\d+\.\d+\.\d+\-hotfix\-x/
+    - /\d+\.\d+\.\d+-hotfix/
+    - /^gh-readonly-queue.*/
 git:
   enable: true
 github:
@@ -15,11 +23,3 @@ code_artifact:
   enable: true
   package_paths:
     - maven-snapshots/maven/io.confluent/snowflake-ingest-sdk
-branches:
-  - master
-  - main
-  - 2.1.0-hotfix
-  - /^\d+\.\d+\.x$/
-  - /v\d+\.\d+\.\d+\-hotfix\-x/
-  - /\d+\.\d+\.\d+-hotfix/
-  - /^gh-readonly-queue.*/
\ No newline at end of file

From 613f4b2d0af87fb62a44f9c2e1419185abaabd4f Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Fri, 7 Feb 2025 10:32:25 +0530
Subject: [PATCH 342/356] put only standard branches (#25)

---
 .semaphore/project.yml | 3 ---
 service.yml            | 2 --
 2 files changed, 5 deletions(-)

diff --git a/.semaphore/project.yml b/.semaphore/project.yml
index 4bfc36806..0a93cd3b1 100644
--- a/.semaphore/project.yml
+++ b/.semaphore/project.yml
@@ -28,11 +28,8 @@ spec:
       branches:
       - master
       - main
-      - 2.1.0-hotfix
       - /^\d+\.\d+\.x$/
       - /v\d+\.\d+\.\d+\-hotfix\-x/
-      - /^\d+\.\d+\.\d+-hotfix$/
-      - /\d+\.\d+\.\d+-hotfix/
       - /^gh-readonly-queue.*/
   custom_permissions: true
   debug_permissions:
diff --git a/service.yml b/service.yml
index 2b79b47ec..d0e4dde4d 100644
--- a/service.yml
+++ b/service.yml
@@ -10,10 +10,8 @@ semaphore:
   branches:
     - master
     - main
-    - 2.1.0-hotfix
     - /^\d+\.\d+\.x$/
     - /v\d+\.\d+\.\d+\-hotfix\-x/
-    - /\d+\.\d+\.\d+-hotfix/
     - /^gh-readonly-queue.*/
 git:
   enable: true

From 4314710d12e3e3d7e99d1d9532468c5a7fb02d14 Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Fri, 7 Feb 2025 11:46:13 +0530
Subject: [PATCH 343/356] remove extra space from README (#26)

* remove extra space from README

* set java version to 8 and remove cve scan
---
 .semaphore/semaphore.yml | 2 +-
 README.md                | 2 +-
 service.yml              | 1 -
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
index da45b7663..d6cb73bd5 100644
--- a/.semaphore/semaphore.yml
+++ b/.semaphore/semaphore.yml
@@ -24,7 +24,7 @@ global_job_config:
   prologue:
     commands:
       - checkout
-      - . set-cp-java-version
+      - sem-version java 8
       - . cache-maven restore
 
 blocks:
diff --git a/README.md b/README.md
index 92015a17d..e0cd0d8b1 100644
--- a/README.md
+++ b/README.md
@@ -119,7 +119,7 @@ mvn install
 ```
 
 If you would just like to build the jar in the source directory, you can
-run 
+run
 
 ``` {.bash}
 mvn package
diff --git a/service.yml b/service.yml
index d0e4dde4d..ec8002f66 100644
--- a/service.yml
+++ b/service.yml
@@ -6,7 +6,6 @@ codeowners:
 semaphore:
   enable: true
   pipeline_type: cp
-  cve_scan: true
   branches:
     - master
     - main

From 35ff91c8a7469fc2755bc3703b33b8c6040afe2d Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Fri, 7 Feb 2025 16:01:12 +0530
Subject: [PATCH 344/356] unskip maven-deploy-plugin, add ci tools update
 version and release tag, add cve-scan (#28)

---
 .semaphore/semaphore.yml | 5 +++++
 CHANGELOG.md             | 0
 README.md                | 2 +-
 pom.xml                  | 1 -
 4 files changed, 6 insertions(+), 2 deletions(-)
 create mode 100644 CHANGELOG.md

diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
index d6cb73bd5..29a8f1952 100644
--- a/.semaphore/semaphore.yml
+++ b/.semaphore/semaphore.yml
@@ -39,6 +39,7 @@ blocks:
           commands:
             - . sem-pint -c
             - mvn -U -Dmaven.wagon.http.retryHandler.count=10 --batch-mode --no-transfer-progress clean verify install dependency:analyze validate
+            - cve-scan
             - . cache-maven store
       epilogue:
         always:
@@ -49,10 +50,14 @@ blocks:
 
   - name: Release
     dependencies: ["Test"]
+    run:
+      when: "branch = 'master' or branch =~ '[0-9]+\\.[0-9]+\\.x' or branch =~ 'v[0-9]+\\.[0-9]+\\.[0-9]+-hotfix-x'"
     task:
       jobs:
         - name: Release
           commands:
+            - ci-tools ci-update-version
+            - ci-tools ci-push-tag
             - mvn -U -Dmaven.wagon.http.retryHandler.count=10 --batch-mode -DaltDeploymentRepository=confluent-codeartifact-internal::default::https://confluent-519856050701.d.codeartifact.us-west-2.amazonaws.com/maven/maven-snapshots/
               -DrepositoryId=confluent-codeartifact-internal deploy -DskipTests
 
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 000000000..e69de29bb
diff --git a/README.md b/README.md
index e0cd0d8b1..92015a17d 100644
--- a/README.md
+++ b/README.md
@@ -119,7 +119,7 @@ mvn install
 ```
 
 If you would just like to build the jar in the source directory, you can
-run
+run 
 
 ``` {.bash}
 mvn package
diff --git a/pom.xml b/pom.xml
index 0d9ffa59d..df5a158fb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -532,7 +532,6 @@
         <plugin>
           <artifactId>maven-deploy-plugin</artifactId>
           <configuration>
-            <skip>true</skip>
           </configuration>
         </plugin>
         <plugin>

From d4df816649fe1697f4e3d635a8db34db9248fadf Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Tue, 11 Feb 2025 23:08:22 +0530
Subject: [PATCH 345/356] add log line (#30)

---
 .../net/snowflake/ingest/SimpleIngestManager.java   | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
index b5462544d..534491aa8 100644
--- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
+++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
@@ -194,6 +194,7 @@ public SimpleIngestManager build() {
   private static final Logger LOGGER = LoggerFactory.getLogger(SimpleIngestManager.class);
   // HTTP Client that we use for sending requests to the service
   private CloseableHttpClient httpClient;
+  private boolean httpClientClosed = false;
 
   // the account in which the user lives
   private String account;
@@ -608,10 +609,14 @@ public HistoryRangeResponse getHistoryRange(
   @Override
   public void close() {
     builder.closeResources();
-    try {
-      httpClient.close();
-    } catch (IOException e) {
-      LOGGER.error("Error closing http client", e);
+    if (!httpClientClosed) {
+      try {
+        LOGGER.info("Closing http client" + Thread.currentThread().getName() + " " + Integer.toHexString(System.identityHashCode(this)));
+        httpClient.close();
+      } catch (IOException e) {
+        LOGGER.error("Error closing http client", e);
+      }
+      httpClientClosed = true;
     }
     HttpUtil.shutdownHttpConnectionManagerDaemonThread();
   }

From 237ed06a8f2b815c34d6f31ba8bc02910d8b0ad4 Mon Sep 17 00:00:00 2001
From: Nirut Gupta <47895961+nirutgupta@users.noreply.github.com>
Date: Tue, 18 Feb 2025 11:03:13 +0530
Subject: [PATCH 346/356] Revert httpclient connection close (#32)

---
 .../java/net/snowflake/ingest/SimpleIngestManager.java   | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
index 534491aa8..8df7e1727 100644
--- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
+++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
@@ -609,15 +609,6 @@ public HistoryRangeResponse getHistoryRange(
   @Override
   public void close() {
     builder.closeResources();
-    if (!httpClientClosed) {
-      try {
-        LOGGER.info("Closing http client" + Thread.currentThread().getName() + " " + Integer.toHexString(System.identityHashCode(this)));
-        httpClient.close();
-      } catch (IOException e) {
-        LOGGER.error("Error closing http client", e);
-      }
-      httpClientClosed = true;
-    }
     HttpUtil.shutdownHttpConnectionManagerDaemonThread();
   }
 

From 75ea7fe8abc0adab5eed8effac0d295fa8527c87 Mon Sep 17 00:00:00 2001
From: Sangeet Mishra <mail2sangeetmishra@gmail.com>
Date: Mon, 9 Jun 2025 13:17:58 +0530
Subject: [PATCH 347/356] CC-33579 - use connector level proxy configs and make
 HttpClient keyed on proxy (#35)

* add proxy support to HTTP client and related classes

* review comments

* convert lamda to method reference

* update groupId and version
---
 pom.xml                                       |   2 +-
 .../snowflake/ingest/SimpleIngestManager.java |  43 +++++-
 .../streaming/internal/FlushService.java      |   3 +-
 ...nowflakeStreamingIngestClientInternal.java |  37 ++++-
 .../internal/StreamingIngestStage.java        |  27 +++-
 .../ingest/utils/HttpClientSettingsKey.java   | 114 +++++++++++++++
 .../net/snowflake/ingest/utils/HttpUtil.java  | 135 ++++++++++++++----
 .../net/snowflake/ingest/utils/Utils.java     |  20 ++-
 .../ingest/utils/HttpUtilCacheTest.java       | 105 ++++++++++++++
 9 files changed, 453 insertions(+), 33 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/utils/HttpClientSettingsKey.java
 create mode 100644 src/test/java/net/snowflake/ingest/utils/HttpUtilCacheTest.java

diff --git a/pom.xml b/pom.xml
index df5a158fb..9f01aa0b6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
   <!-- Arifact name and version information -->
   <groupId>io.confluent</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.1.0-hotfix1</version>
+  <version>2.1.1</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
index 8df7e1727..cdabebc42 100644
--- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
+++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
@@ -15,6 +15,7 @@
 import java.security.spec.InvalidKeySpecException;
 import java.util.Collections;
 import java.util.List;
+import java.util.Properties;
 import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
@@ -385,6 +386,30 @@ public SimpleIngestManager(
         new RequestBuilder(account, user, keyPair, schemeName, hostName, port, userAgentSuffix);
   }
 
+
+  /* Another flavor of constructor which supports userAgentSuffix and proxyProperties */
+  public SimpleIngestManager(
+    String account,
+    String user,
+    String pipe,
+    PrivateKey privateKey,
+    String schemeName,
+    String hostName,
+    int port,
+    String userAgentSuffix,
+    Properties proxyProperties)
+    throws NoSuchAlgorithmException, InvalidKeySpecException {
+  KeyPair keyPair = Utils.createKeyPairFromPrivateKey(privateKey);
+  // call our initializer method
+  init(account, user, pipe, keyPair, proxyProperties);
+
+  // make the request builder we'll use to build messages to the service
+  builder =
+      new RequestBuilder(account, user, keyPair, schemeName, hostName, port, userAgentSuffix);
+}
+
+  
+
   // ========= Constructors End =========
 
   /**
@@ -397,14 +422,28 @@ public SimpleIngestManager(
    * @param keyPair the KeyPair we'll use to sign JWT tokens
    */
   private void init(String account, String user, String pipe, KeyPair keyPair) {
+    init(account, user, pipe, keyPair, new Properties());
+  }
+
+  /**
+   * init - Does the basic work of constructing a SimpleIngestManager that is common across all
+   * constructors
+   *
+   * @param account The account into which we're loading
+   * @param user the user performing this load
+   * @param pipe the fully qualified name of pipe
+   * @param keyPair the KeyPair we'll use to sign JWT tokens
+   * @param proxyProperties proxy properties for HTTP client configuration
+   */
+  private void init(String account, String user, String pipe, KeyPair keyPair, Properties proxyProperties) {
     // set up our reference variables
     this.account = account;
     this.user = user;
     this.pipe = pipe;
     this.keyPair = keyPair;
 
-    // make our client for sending requests
-    httpClient = HttpUtil.getHttpClient(account);
+    // make our client for sending requests with proxy properties support
+    httpClient = HttpUtil.getHttpClient(account, proxyProperties);
     // make the request builder we'll use to build messages to the service
   }
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 2324adda8..30dadbe66 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -167,7 +167,8 @@ List<List<ChannelData<T>>> getData() {
               client.getHttpClient(),
               client.getRequestBuilder(),
               client.getName(),
-              DEFAULT_MAX_UPLOAD_RETRIES);
+              DEFAULT_MAX_UPLOAD_RETRIES,
+              client.getProxyProperties());
     } catch (SnowflakeSQLException | IOException err) {
       throw new SFException(err, ErrorCode.UNABLE_TO_CONNECT_TO_STAGE);
     }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 62fc265ff..454752a02 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -142,6 +142,9 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
   // Background thread that uploads telemetry data periodically
   private ScheduledExecutorService telemetryWorker;
 
+  // Store original properties for proxy configuration
+  private final Properties originalProperties;
+
   /**
    * Constructor
    *
@@ -162,11 +165,12 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
       RequestBuilder requestBuilder,
       Map<String, Object> parameterOverrides) {
     this.parameterProvider = new ParameterProvider(parameterOverrides, prop);
+    this.originalProperties = prop;
 
     this.name = name;
     String accountName = accountURL == null ? null : accountURL.getAccount();
     this.isTestMode = isTestMode;
-    this.httpClient = httpClient == null ? HttpUtil.getHttpClient(accountName) : httpClient;
+    this.httpClient = HttpUtil.getHttpClient(accountName, prop);
     this.channelCache = new ChannelCache<>();
     this.isClosed = false;
     this.requestBuilder = requestBuilder;
@@ -1075,4 +1079,35 @@ private void cleanUpResources() {
       HttpUtil.shutdownHttpConnectionManagerDaemonThread();
     }
   }
+
+  /**
+   * Extract proxy properties from the original Properties object
+   *
+   * @return Properties containing proxy configuration
+   */
+  Properties getProxyProperties() {
+    Properties proxyProperties = new Properties();
+
+    
+    if (this.originalProperties != null) {
+
+      for (String key : this.originalProperties.stringPropertyNames()) {
+        if (key.equals(SFSessionProperty.USE_PROXY.getPropertyKey()) ||
+            key.equals(SFSessionProperty.PROXY_HOST.getPropertyKey()) ||
+            key.equals(SFSessionProperty.PROXY_PORT.getPropertyKey()) ||
+            key.equals(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey()) ||
+            key.equals(SFSessionProperty.PROXY_USER.getPropertyKey()) ||
+            key.equals(SFSessionProperty.PROXY_PASSWORD.getPropertyKey())) {
+          proxyProperties.put(key, this.originalProperties.getProperty(key));
+        }
+      }
+    }
+    
+    // If no proxy properties found in original properties, fall back to system properties
+    if (proxyProperties.isEmpty()) {
+      return HttpUtil.generateProxyPropertiesForJDBC();
+    }
+    
+    return proxyProperties;
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index e8e56f383..e707d3b07 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -98,11 +98,34 @@ state to record unknown age.
       String clientName,
       int maxUploadRetries)
       throws SnowflakeSQLException, IOException {
+    this(isTestMode, role, httpClient, requestBuilder, clientName, maxUploadRetries, null);
+  }
+
+  /**
+   * Constructor with proxy properties support
+   *
+   * @param isTestMode whether we're under test mode
+   * @param role Snowflake role used by the Client
+   * @param httpClient http client reference
+   * @param requestBuilder request builder to build the HTTP request
+   * @param clientName the client name
+   * @param maxUploadRetries maximum number of upload retries
+   * @param proxyProperties proxy properties for HTTP client configuration
+   */
+  StreamingIngestStage(
+      boolean isTestMode,
+      String role,
+      CloseableHttpClient httpClient,
+      RequestBuilder requestBuilder,
+      String clientName,
+      int maxUploadRetries,
+      Properties proxyProperties)
+      throws SnowflakeSQLException, IOException {
     this.httpClient = httpClient;
     this.role = role;
     this.requestBuilder = requestBuilder;
     this.clientName = clientName;
-    this.proxyProperties = generateProxyPropertiesForJDBC();
+    this.proxyProperties = proxyProperties != null ? proxyProperties : generateProxyPropertiesForJDBC();
     this.maxUploadRetries = maxUploadRetries;
 
     if (!isTestMode) {
@@ -129,7 +152,7 @@ state to record unknown age.
       SnowflakeFileTransferMetadataWithAge testMetadata,
       int maxRetryCount)
       throws SnowflakeSQLException, IOException {
-    this(isTestMode, role, httpClient, requestBuilder, clientName, maxRetryCount);
+    this(isTestMode, role, httpClient, requestBuilder, clientName, maxRetryCount, null);
     if (!isTestMode) {
       throw new SFException(ErrorCode.INTERNAL_ERROR);
     }
diff --git a/src/main/java/net/snowflake/ingest/utils/HttpClientSettingsKey.java b/src/main/java/net/snowflake/ingest/utils/HttpClientSettingsKey.java
new file mode 100644
index 000000000..149a3a29e
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/utils/HttpClientSettingsKey.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2012-2017 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.utils;
+
+import static net.snowflake.ingest.utils.Utils.isNullOrEmpty;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * This class defines all parameters needed to create an HttpClient object for the ingest service.
+ * It is used as the key for the static hashmap of reusable http clients.
+ */
+public class HttpClientSettingsKey implements Serializable {
+
+  private boolean useProxy;
+  private String proxyHost = "";
+  private int proxyPort = 0;
+  private String nonProxyHosts = "";
+  private String proxyUser = "";
+  private String proxyPassword = "";
+  private String accountName = "";
+
+  /**
+   * Constructor for proxy configuration
+   */
+  public HttpClientSettingsKey(
+      String accountName,
+      String proxyHost,
+      int proxyPort,
+      String nonProxyHosts,
+      String proxyUser,
+      String proxyPassword) {
+    this.useProxy = true;
+    this.accountName = !isNullOrEmpty(accountName) ? accountName.trim() : "";
+    this.proxyHost = !isNullOrEmpty(proxyHost) ? proxyHost.trim() : "";
+    this.proxyPort = proxyPort;
+    this.nonProxyHosts = !isNullOrEmpty(nonProxyHosts) ? nonProxyHosts.trim() : "";
+    this.proxyUser = !isNullOrEmpty(proxyUser) ? proxyUser.trim() : "";
+    this.proxyPassword = !isNullOrEmpty(proxyPassword) ? proxyPassword.trim() : "";
+  }
+
+  /**
+   * Constructor for non-proxy configuration
+   */
+  public HttpClientSettingsKey(String accountName) {
+    this.useProxy = false;
+    this.accountName = !isNullOrEmpty(accountName) ? accountName.trim() : "";
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) return true;
+    if (obj == null || getClass() != obj.getClass()) return false;
+    
+    HttpClientSettingsKey that = (HttpClientSettingsKey) obj;
+    
+    return useProxy == that.useProxy &&
+           proxyPort == that.proxyPort &&
+           Objects.equals(accountName, that.accountName) &&
+           Objects.equals(proxyHost, that.proxyHost) &&
+           Objects.equals(nonProxyHosts, that.nonProxyHosts) &&
+           Objects.equals(proxyUser, that.proxyUser) &&
+           Objects.equals(proxyPassword, that.proxyPassword);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(useProxy, accountName, proxyHost, proxyPort, nonProxyHosts, proxyUser, proxyPassword);
+  }
+
+  public boolean usesProxy() {
+    return this.useProxy;
+  }
+
+  public String getAccountName() {
+    return this.accountName;
+  }
+
+  public String getProxyHost() {
+    return this.proxyHost;
+  }
+
+  public int getProxyPort() {
+    return this.proxyPort;
+  }
+
+  public String getProxyUser() {
+    return this.proxyUser;
+  }
+
+  public String getProxyPassword() {
+    return this.proxyPassword;
+  }
+
+  public String getNonProxyHosts() {
+    return this.nonProxyHosts;
+  }
+
+  @Override
+  public String toString() {
+    return "HttpClientSettingsKey[" +
+           "accountName='" + accountName + '\'' +
+           ", useProxy=" + useProxy +
+           ", proxyHost='" + proxyHost + '\'' +
+           ", proxyPort=" + proxyPort +
+           ", nonProxyHosts='" + nonProxyHosts + '\'' +
+           ", proxyUser='" + proxyUser + '\'' +
+           ", proxyPassword=" + (proxyPassword.isEmpty() ? "not set" : "set") +
+           ']';
+  }
+} 
diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index 1ff65a095..03f8b05eb 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -10,8 +10,10 @@
 import java.time.Duration;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.regex.Pattern;
@@ -66,7 +68,8 @@ public class HttpUtil {
    */
   private static final int MAX_RETRIES = 10;
 
-  private static volatile CloseableHttpClient httpClient;
+  private static final Map<HttpClientSettingsKey, CloseableHttpClient> httpClientCache =
+      new ConcurrentHashMap<>();
 
   private static PoolingHttpClientConnectionManager connectionManager;
 
@@ -100,28 +103,83 @@ public class HttpUtil {
   // Only connections that are currently owned, not checked out, are subject to idle timeouts.
   private static final int DEFAULT_IDLE_CONNECTION_TIMEOUT_SECONDS = 30;
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(HttpUtil.class);
+
   /**
-   * @param {@code String} account name to connect to (excluding snowflakecomputing.com domain)
+   * Get HttpClient with proxy properties support
+   *
+   * @param accountName account name to connect to (excluding snowflakecomputing.com domain)
+   * @param proxyProperties proxy properties, can be null
    * @return Instance of CloseableHttpClient
    */
-  public static CloseableHttpClient getHttpClient(String accountName) {
-    if (httpClient == null) {
-      synchronized (HttpUtil.class) {
-        if (httpClient == null) {
-          initHttpClient(accountName);
-        }
-      }
-    }
+  public static CloseableHttpClient getHttpClient(String accountName, Properties proxyProperties) {
 
-    initIdleConnectionMonitoringThread();
+    HttpClientSettingsKey key = createHttpClientSettingsKey(accountName, proxyProperties);
 
-    return httpClient;
+    return httpClientCache.computeIfAbsent(key, HttpUtil::buildHttpClient);
   }
 
-  private static final Logger LOGGER = LoggerFactory.getLogger(HttpUtil.class);
+  /**
+   * @param accountName account name to connect to (excluding snowflakecomputing.com domain)
+   * @return Instance of CloseableHttpClient
+   */
+  public static CloseableHttpClient getHttpClient(String accountName) {
+    return getHttpClient(accountName, null);
+  }
+
+  /**
+   * Create HttpClientSettingsKey from account name and proxy properties
+   */
+  private static HttpClientSettingsKey createHttpClientSettingsKey(String accountName, Properties proxyProperties) {
 
-  private static void initHttpClient(String accountName) {
+    if (proxyProperties != null && proxyProperties.containsKey(SFSessionProperty.USE_PROXY.getPropertyKey())) {
 
+      Boolean useProxy = Boolean.valueOf(proxyProperties.getProperty(SFSessionProperty.USE_PROXY.getPropertyKey()));
+
+      if (useProxy) {
+        String proxyHost = proxyProperties.getProperty(SFSessionProperty.PROXY_HOST.getPropertyKey(), "");
+        String proxyPortStr = proxyProperties.getProperty(SFSessionProperty.PROXY_PORT.getPropertyKey(), "0");
+        int proxyPort = 0;
+        try {
+          proxyPort = Integer.parseInt(proxyPortStr);
+        } catch (NumberFormatException e) {
+          throw new IllegalArgumentException(
+              "Invalid proxy port: '" + proxyPortStr + "'. Proxy port must be a valid integer.", e);
+        }
+        String nonProxyHosts = proxyProperties.getProperty(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey(), "");
+        String proxyUser = proxyProperties.getProperty(SFSessionProperty.PROXY_USER.getPropertyKey(), "");
+        String proxyPassword = proxyProperties.getProperty(SFSessionProperty.PROXY_PASSWORD.getPropertyKey(), "");
+        
+        return new HttpClientSettingsKey(accountName, proxyHost, proxyPort, nonProxyHosts, proxyUser, proxyPassword);
+      }
+    }
+    
+    // Check system properties for proxy configuration (backward compatibility)
+    if ("true".equalsIgnoreCase(System.getProperty(USE_PROXY)) && !shouldBypassProxy(accountName)) {
+      String proxyHost = System.getProperty(PROXY_HOST, "");
+      String proxyPortStr = System.getProperty(PROXY_PORT, "0");
+      int proxyPort = 0;
+      try {
+        proxyPort = Integer.parseInt(proxyPortStr);
+      } catch (NumberFormatException e) {
+        throw new IllegalArgumentException(
+            "Invalid proxy port: '" + proxyPortStr + "'. Proxy port must be a valid integer.", e);
+      }
+      String nonProxyHosts = System.getProperty(NON_PROXY_HOSTS, "");
+      String proxyUser = System.getProperty(HTTP_PROXY_USER, "");
+      String proxyPassword = System.getProperty(HTTP_PROXY_PASSWORD, "");
+      
+      return new HttpClientSettingsKey(accountName, proxyHost, proxyPort, nonProxyHosts, proxyUser, proxyPassword);
+    }
+    
+    // No proxy configuration
+    return new HttpClientSettingsKey(accountName);
+  }
+
+  /**
+   * Build HttpClient based on the settings key
+   */
+  private static CloseableHttpClient buildHttpClient(HttpClientSettingsKey key) {
     Security.setProperty("ocsp.enable", "true");
 
     SSLContext sslContext = SSLContexts.createDefault();
@@ -167,24 +225,25 @@ private static void initHttpClient(String accountName) {
             .setDefaultRequestConfig(requestConfig);
 
     // proxy settings
-    if ("true".equalsIgnoreCase(System.getProperty(USE_PROXY)) && !shouldBypassProxy(accountName)) {
-      if (System.getProperty(PROXY_PORT) == null) {
+    if (key.usesProxy()) {
+      if (isNullOrEmpty(key.getProxyHost())) {
         throw new IllegalArgumentException(
-            "proxy port number is not provided, please assign proxy port to http.proxyPort option");
+            "proxy host IP is not provided, please assign proxy host IP to http.proxyHost option");
       }
-      if (System.getProperty(PROXY_HOST) == null) {
+      if (key.getProxyPort() <= 0) {
         throw new IllegalArgumentException(
-            "proxy host IP is not provided, please assign proxy host IP to http.proxyHost option");
+            "proxy port number is not provided, please assign proxy port to http.proxyPort option");
       }
-      String proxyHost = System.getProperty(PROXY_HOST);
-      int proxyPort = Integer.parseInt(System.getProperty(PROXY_PORT));
+      
+      String proxyHost = key.getProxyHost();
+      int proxyPort = key.getProxyPort();
       HttpHost proxy = new HttpHost(proxyHost, proxyPort, PROXY_SCHEME);
       DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
       clientBuilder = clientBuilder.setRoutePlanner(routePlanner);
 
       // Check if proxy username and password are set
-      final String proxyUser = System.getProperty(HTTP_PROXY_USER);
-      final String proxyPassword = System.getProperty(HTTP_PROXY_PASSWORD);
+      final String proxyUser = key.getProxyUser();
+      final String proxyPassword = key.getProxyPassword();
       if (!isNullOrEmpty(proxyUser) && !isNullOrEmpty(proxyPassword)) {
         Credentials credentials = new UsernamePasswordCredentials(proxyUser, proxyPassword);
         AuthScope authScope = new AuthScope(proxyHost, proxyPort);
@@ -194,7 +253,9 @@ private static void initHttpClient(String accountName) {
       }
     }
 
-    httpClient = clientBuilder.build();
+    CloseableHttpClient httpClient = clientBuilder.build();
+    initIdleConnectionMonitoringThread();
+    return httpClient;
   }
 
   /** Starts a daemon thread to monitor idle connections in http connection manager. */
@@ -420,7 +481,31 @@ private boolean isShutdown() {
 
   /** Shuts down the daemon thread. */
   public static void shutdownHttpConnectionManagerDaemonThread() {
-    idleConnectionMonitorThread.shutdown();
+    idleConnectionMonitorThreadLock.lock();
+    try {
+      if (idleConnectionMonitorThread != null) {
+        idleConnectionMonitorThread.shutdown();
+        idleConnectionMonitorThread = null;
+      }
+    } finally {
+      idleConnectionMonitorThreadLock.unlock();
+    }
+  }
+
+  /**
+   * Close all cached HTTP clients and clear the cache
+   */
+  public static void closeAllHttpClients() {
+    LOGGER.debug("Closing all cached HTTP clients");
+    for (Map.Entry<HttpClientSettingsKey, CloseableHttpClient> entry : httpClientCache.entrySet()) {
+      try {
+        entry.getValue().close();
+      } catch (Exception e) {
+        LOGGER.warn("Error closing HTTP client for key: {}", entry.getKey(), e);
+      }
+    }
+    httpClientCache.clear();
+    shutdownHttpConnectionManagerDaemonThread();
   }
 
   /** Create Pool stats for a route */
diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java
index a2653ab14..a9deb682b 100644
--- a/src/main/java/net/snowflake/ingest/utils/Utils.java
+++ b/src/main/java/net/snowflake/ingest/utils/Utils.java
@@ -139,7 +139,12 @@ public static Properties createProperties(Properties inputProp) {
           privateKeyPassphrase = val;
           break;
         default:
-          properties.put(key.toLowerCase(), val);
+          // Preserve case for proxy-related properties that HttpUtil expects
+          if (isProxyRelatedProperty(key)) {
+            properties.put(key, val);
+          } else {
+            properties.put(key.toLowerCase(), val);
+          }
       }
     }
 
@@ -215,6 +220,19 @@ public static Properties createProperties(Properties inputProp) {
     return properties;
   }
 
+  /**
+   * Check if a property key is proxy-related and should preserve its case
+   */
+  private static boolean isProxyRelatedProperty(String key) {
+    return key.equals(SFSessionProperty.USE_PROXY.getPropertyKey()) ||
+           key.equals(SFSessionProperty.PROXY_HOST.getPropertyKey()) ||
+           key.equals(SFSessionProperty.PROXY_PORT.getPropertyKey()) ||
+           key.equals(SFSessionProperty.PROXY_USER.getPropertyKey()) ||
+           key.equals(SFSessionProperty.PROXY_PASSWORD.getPropertyKey()) ||
+           key.equals(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey()) ||
+           key.equals(SFSessionProperty.PROXY_PROTOCOL.getPropertyKey());
+  }
+
   /** Construct account url from input schema, host and port */
   public static String constructAccountUrl(String scheme, String host, int port) {
     return String.format("%s://%s:%d", scheme, host, port);
diff --git a/src/test/java/net/snowflake/ingest/utils/HttpUtilCacheTest.java b/src/test/java/net/snowflake/ingest/utils/HttpUtilCacheTest.java
new file mode 100644
index 000000000..167930dce
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/utils/HttpUtilCacheTest.java
@@ -0,0 +1,105 @@
+package net.snowflake.ingest.utils;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+
+import java.util.Properties;
+import net.snowflake.client.core.SFSessionProperty;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
+import org.junit.After;
+import org.junit.Test;
+
+/**
+ * Test class for HttpUtil caching functionality.
+ * Tests verify that HTTP clients are properly cached based on account name and proxy settings.
+ * Note: Both JDBC and streaming clients now use unified SFSessionProperty constants for proxy configuration.
+ */
+public class HttpUtilCacheTest {
+
+  @After
+  public void tearDown() {
+    // Clean up after each test
+    HttpUtil.closeAllHttpClients();
+  }
+
+  @Test
+  public void testHttpClientCachingWithSameAccount() {
+    String accountName = "testaccount";
+    
+    // Get HTTP client twice with same account
+    CloseableHttpClient client1 = HttpUtil.getHttpClient(accountName);
+    CloseableHttpClient client2 = HttpUtil.getHttpClient(accountName);
+    
+    // Should return the same instance
+    assertSame("HTTP clients should be the same for same account", client1, client2);
+  }
+
+  @Test
+  public void testHttpClientCachingWithDifferentAccounts() {
+    String accountName1 = "testaccount1";
+    String accountName2 = "testaccount2";
+    
+    // Get HTTP clients for different accounts
+    CloseableHttpClient client1 = HttpUtil.getHttpClient(accountName1);
+    CloseableHttpClient client2 = HttpUtil.getHttpClient(accountName2);
+    
+    // Should return different instances
+    assertNotSame("HTTP clients should be different for different accounts", client1, client2);
+  }
+
+  @Test
+  public void testHttpClientCachingWithSameProxySettings() {
+    String accountName = "testaccount";
+    Properties proxyProps = new Properties();
+    proxyProps.setProperty(SFSessionProperty.USE_PROXY.getPropertyKey(), "true");
+    proxyProps.setProperty(SFSessionProperty.PROXY_HOST.getPropertyKey(), "proxy.example.com");
+    proxyProps.setProperty(SFSessionProperty.PROXY_PORT.getPropertyKey(), "8080");
+    
+    // Get HTTP client twice with same proxy settings
+    CloseableHttpClient client1 = HttpUtil.getHttpClient(accountName, proxyProps);
+    CloseableHttpClient client2 = HttpUtil.getHttpClient(accountName, proxyProps);
+    
+    // Should return the same instance
+    assertSame("HTTP clients should be the same for same proxy settings", client1, client2);
+  }
+
+  @Test
+  public void testHttpClientCachingWithDifferentProxySettings() {
+    String accountName = "testaccount";
+    
+    Properties proxyProps1 = new Properties();
+    proxyProps1.setProperty(SFSessionProperty.USE_PROXY.getPropertyKey(), "true");
+    proxyProps1.setProperty(SFSessionProperty.PROXY_HOST.getPropertyKey(), "proxy1.example.com");
+    proxyProps1.setProperty(SFSessionProperty.PROXY_PORT.getPropertyKey(), "8080");
+    
+    Properties proxyProps2 = new Properties();
+    proxyProps2.setProperty(SFSessionProperty.USE_PROXY.getPropertyKey(), "true");
+    proxyProps2.setProperty(SFSessionProperty.PROXY_HOST.getPropertyKey(), "proxy2.example.com");
+    proxyProps2.setProperty(SFSessionProperty.PROXY_PORT.getPropertyKey(), "8080");
+    
+    // Get HTTP clients with different proxy settings
+    CloseableHttpClient client1 = HttpUtil.getHttpClient(accountName, proxyProps1);
+    CloseableHttpClient client2 = HttpUtil.getHttpClient(accountName, proxyProps2);
+    
+    // Should return different instances
+    assertNotSame("HTTP clients should be different for different proxy settings", client1, client2);
+  }
+
+  @Test
+  public void testHttpClientCachingWithProxyAndNoProxy() {
+    String accountName = "testaccount";
+    
+    Properties proxyProps = new Properties();
+    proxyProps.setProperty(SFSessionProperty.USE_PROXY.getPropertyKey(), "true");
+    proxyProps.setProperty(SFSessionProperty.PROXY_HOST.getPropertyKey(), "proxy.example.com");
+    proxyProps.setProperty(SFSessionProperty.PROXY_PORT.getPropertyKey(), "8080");
+    
+    // Get HTTP client with proxy and without proxy
+    CloseableHttpClient clientWithProxy = HttpUtil.getHttpClient(accountName, proxyProps);
+    CloseableHttpClient clientWithoutProxy = HttpUtil.getHttpClient(accountName);
+    
+    // Should return different instances
+    assertNotSame("HTTP clients should be different for proxy vs no proxy", clientWithProxy, clientWithoutProxy);
+  }
+} 

From 483cacd7b54ef5c2c4c7cf1f8dca764c4b53fc4e Mon Sep 17 00:00:00 2001
From: Sangeet Mishra <mail2sangeetmishra@gmail.com>
Date: Mon, 9 Jun 2025 13:30:39 +0530
Subject: [PATCH 348/356] update machine to ubuntu24 (#39)

---
 .semaphore/semaphore.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
index 29a8f1952..4968487b6 100644
--- a/.semaphore/semaphore.yml
+++ b/.semaphore/semaphore.yml
@@ -7,7 +7,7 @@ version: v1.0
 name: build-test-release
 agent:
   machine:
-    type: s1-prod-ubuntu20-04-amd64-1
+    type: s1-prod-ubuntu24-04-amd64-1
 
 fail_fast:
   cancel:

From 8fbe2c597674cbe17e83e875032a001de8706e39 Mon Sep 17 00:00:00 2001
From: Sangeet Mishra <mail2sangeetmishra@gmail.com>
Date: Mon, 9 Jun 2025 15:38:01 +0530
Subject: [PATCH 349/356] Update sem machine to ubuntu 24 (#40)

* update machine to ubuntu24

* fix tests and add logs

* fix tests
---
 .../snowflake/ingest/SimpleIngestManager.java |  6 +++
 .../ingest/connection/RequestBuilder.java     |  2 +-
 ...nowflakeStreamingIngestClientInternal.java | 36 ++++++++++++-
 .../ingest/utils/HttpClientSettingsKey.java   | 11 ++++
 .../net/snowflake/ingest/utils/HttpUtil.java  | 50 ++++++++++++++++++-
 5 files changed, 101 insertions(+), 4 deletions(-)

diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
index cdabebc42..e6804af90 100644
--- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
+++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
@@ -443,6 +443,12 @@ private void init(String account, String user, String pipe, KeyPair keyPair, Pro
     this.keyPair = keyPair;
 
     // make our client for sending requests with proxy properties support
+    if (proxyProperties != null && !proxyProperties.isEmpty()) {
+      LOGGER.info("Creating HTTP client for SimpleIngestManager with proxy properties for account: {}, user: {}", account, user);
+    } else {
+      LOGGER.info("Creating HTTP client for SimpleIngestManager without proxy properties for account: {}, user: {}", account, user);
+    }
+    
     httpClient = HttpUtil.getHttpClient(account, proxyProperties);
     // make the request builder we'll use to build messages to the service
   }
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 8e12d925e..e44d82c14 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.1.0-hotfix1";
+  public static final String DEFAULT_VERSION = "2.1.1";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 454752a02..51e6a4d86 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -170,7 +170,28 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
     this.name = name;
     String accountName = accountURL == null ? null : accountURL.getAccount();
     this.isTestMode = isTestMode;
-    this.httpClient = HttpUtil.getHttpClient(accountName, prop);
+    
+    if (prop != null && !prop.isEmpty()) {
+      // Check if proxy-related properties are present
+      boolean hasProxyConfig = prop.stringPropertyNames().stream()
+          .anyMatch(key -> key.startsWith("http.proxy") || 
+                          key.equals("useProxy") || 
+                          key.equals("proxyHost") || 
+                          key.equals("proxyPort") ||
+                          key.equals("nonProxyHosts") ||
+                          key.equals("proxyUser") ||
+                          key.equals("proxyPassword"));
+      
+      if (hasProxyConfig) {
+        logger.logInfo("Creating HTTP client for SnowflakeStreamingIngestClient with proxy configuration for account: {}, client: {}", accountName, name);
+      } else {
+        logger.logInfo("Creating HTTP client for SnowflakeStreamingIngestClient without proxy configuration for account: {}, client: {}", accountName, name);
+      }
+    } else {
+      logger.logInfo("Creating HTTP client for SnowflakeStreamingIngestClient with no properties for account: {}, client: {}", accountName, name);
+    }
+    
+    this.httpClient = (httpClient != null) ? httpClient : HttpUtil.getHttpClient(accountName, prop);
     this.channelCache = new ChannelCache<>();
     this.isClosed = false;
     this.requestBuilder = requestBuilder;
@@ -1101,10 +1122,23 @@ Properties getProxyProperties() {
           proxyProperties.put(key, this.originalProperties.getProperty(key));
         }
       }
+      
+      if (!proxyProperties.isEmpty()) {
+        logger.logInfo("Extracted {} proxy properties from original properties for client: {}", proxyProperties.size(), this.name);
+        logger.logDebug("Proxy properties extracted: {}", 
+                       proxyProperties.keySet().stream()
+                           .map(k -> k + "=" + (k.toString().toLowerCase().contains("password") ? "[HIDDEN]" : proxyProperties.get(k)))
+                           .collect(Collectors.joining(", ")));
+      } else {
+        logger.logInfo("No proxy properties found in original properties for client: {}", this.name);
+      }
+    } else {
+      logger.logInfo("Original properties is null, cannot extract proxy properties for client: {}", this.name);
     }
     
     // If no proxy properties found in original properties, fall back to system properties
     if (proxyProperties.isEmpty()) {
+      logger.logInfo("Falling back to system properties for proxy configuration for client: {}", this.name);
       return HttpUtil.generateProxyPropertiesForJDBC();
     }
     
diff --git a/src/main/java/net/snowflake/ingest/utils/HttpClientSettingsKey.java b/src/main/java/net/snowflake/ingest/utils/HttpClientSettingsKey.java
index 149a3a29e..b089d81f2 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpClientSettingsKey.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpClientSettingsKey.java
@@ -8,6 +8,8 @@
 
 import java.io.Serializable;
 import java.util.Objects;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This class defines all parameters needed to create an HttpClient object for the ingest service.
@@ -15,6 +17,8 @@
  */
 public class HttpClientSettingsKey implements Serializable {
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(HttpClientSettingsKey.class);
+
   private boolean useProxy;
   private String proxyHost = "";
   private int proxyPort = 0;
@@ -40,6 +44,11 @@ public HttpClientSettingsKey(
     this.nonProxyHosts = !isNullOrEmpty(nonProxyHosts) ? nonProxyHosts.trim() : "";
     this.proxyUser = !isNullOrEmpty(proxyUser) ? proxyUser.trim() : "";
     this.proxyPassword = !isNullOrEmpty(proxyPassword) ? proxyPassword.trim() : "";
+    
+    LOGGER.debug("Created HttpClientSettingsKey with proxy configuration for account: {}. Host: {}, Port: {}, User: {}, NonProxyHosts: {}", 
+                this.accountName, this.proxyHost, this.proxyPort, 
+                !isNullOrEmpty(this.proxyUser) ? "set" : "not set", 
+                !isNullOrEmpty(this.nonProxyHosts) ? this.nonProxyHosts : "not set");
   }
 
   /**
@@ -48,6 +57,8 @@ public HttpClientSettingsKey(
   public HttpClientSettingsKey(String accountName) {
     this.useProxy = false;
     this.accountName = !isNullOrEmpty(accountName) ? accountName.trim() : "";
+    
+    LOGGER.debug("Created HttpClientSettingsKey without proxy configuration for account: {}", this.accountName);
   }
 
   @Override
diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index 03f8b05eb..0c8769f85 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -116,6 +116,14 @@ public static CloseableHttpClient getHttpClient(String accountName, Properties p
 
     HttpClientSettingsKey key = createHttpClientSettingsKey(accountName, proxyProperties);
 
+    CloseableHttpClient client = httpClientCache.get(key);
+    // todo: this is for testing phase
+    if (client != null) {
+      LOGGER.info("Reusing existing HTTP client for account: {}, key: {}", accountName, key);
+      return client;
+    }
+
+    LOGGER.info("No existing HTTP client found for account: {}, key: {}. Creating new HTTP client.", accountName, key);
     return httpClientCache.computeIfAbsent(key, HttpUtil::buildHttpClient);
   }
 
@@ -150,7 +158,11 @@ private static HttpClientSettingsKey createHttpClientSettingsKey(String accountN
         String proxyUser = proxyProperties.getProperty(SFSessionProperty.PROXY_USER.getPropertyKey(), "");
         String proxyPassword = proxyProperties.getProperty(SFSessionProperty.PROXY_PASSWORD.getPropertyKey(), "");
         
+        LOGGER.info("Creating HTTP client settings key with proxy configuration from properties for account: {}. Proxy host: {}, port: {}, user: {}, nonProxyHosts: {}", 
+                   accountName, proxyHost, proxyPort, isNullOrEmpty(proxyUser) ? "not set" : "set", isNullOrEmpty(nonProxyHosts) ? "not set" : nonProxyHosts);
         return new HttpClientSettingsKey(accountName, proxyHost, proxyPort, nonProxyHosts, proxyUser, proxyPassword);
+      } else {
+        LOGGER.info("Creating HTTP client settings key with proxy explicitly disabled via properties for account: {}", accountName);
       }
     }
     
@@ -169,10 +181,13 @@ private static HttpClientSettingsKey createHttpClientSettingsKey(String accountN
       String proxyUser = System.getProperty(HTTP_PROXY_USER, "");
       String proxyPassword = System.getProperty(HTTP_PROXY_PASSWORD, "");
       
+      LOGGER.info("Creating HTTP client settings key with proxy configuration from system properties for account: {}. Proxy host: {}, port: {}, user: {}, nonProxyHosts: {}", 
+                 accountName, proxyHost, proxyPort, isNullOrEmpty(proxyUser) ? "not set" : "set", isNullOrEmpty(nonProxyHosts) ? "not set" : nonProxyHosts);
       return new HttpClientSettingsKey(accountName, proxyHost, proxyPort, nonProxyHosts, proxyUser, proxyPassword);
     }
     
     // No proxy configuration
+    LOGGER.info("Creating HTTP client settings key without proxy configuration for account: {}", accountName);
     return new HttpClientSettingsKey(accountName);
   }
 
@@ -180,6 +195,8 @@ private static HttpClientSettingsKey createHttpClientSettingsKey(String accountN
    * Build HttpClient based on the settings key
    */
   private static CloseableHttpClient buildHttpClient(HttpClientSettingsKey key) {
+    LOGGER.info("Building new HTTP client for key: {}", key);
+    
     Security.setProperty("ocsp.enable", "true");
 
     SSLContext sslContext = SSLContexts.createDefault();
@@ -226,6 +243,8 @@ private static CloseableHttpClient buildHttpClient(HttpClientSettingsKey key) {
 
     // proxy settings
     if (key.usesProxy()) {
+      LOGGER.info("Configuring HTTP client with proxy settings. Host: {}, Port: {}", key.getProxyHost(), key.getProxyPort());
+      
       if (isNullOrEmpty(key.getProxyHost())) {
         throw new IllegalArgumentException(
             "proxy host IP is not provided, please assign proxy host IP to http.proxyHost option");
@@ -245,16 +264,22 @@ private static CloseableHttpClient buildHttpClient(HttpClientSettingsKey key) {
       final String proxyUser = key.getProxyUser();
       final String proxyPassword = key.getProxyPassword();
       if (!isNullOrEmpty(proxyUser) && !isNullOrEmpty(proxyPassword)) {
+        LOGGER.info("Configuring HTTP client with proxy authentication credentials");
         Credentials credentials = new UsernamePasswordCredentials(proxyUser, proxyPassword);
         AuthScope authScope = new AuthScope(proxyHost, proxyPort);
         CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
         credentialsProvider.setCredentials(authScope, credentials);
         clientBuilder = clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+      } else {
+        LOGGER.info("HTTP client configured with proxy but no authentication credentials provided");
       }
+    } else {
+      LOGGER.info("Configuring HTTP client without proxy settings");
     }
 
     CloseableHttpClient httpClient = clientBuilder.build();
     initIdleConnectionMonitoringThread();
+    LOGGER.info("Successfully built new HTTP client for key: {}", key);
     return httpClient;
   }
 
@@ -385,6 +410,8 @@ private static String getRequestUriFromContext(final HttpContext httpContext) {
   public static Properties generateProxyPropertiesForJDBC() {
     Properties proxyProperties = new Properties();
     if (Boolean.parseBoolean(System.getProperty(USE_PROXY))) {
+      LOGGER.info("Generating proxy properties for JDBC from system properties");
+      
       if (isNullOrEmpty(System.getProperty(PROXY_PORT))) {
         throw new IllegalArgumentException(
             "proxy port number is not provided, please assign proxy port to http.proxyPort option");
@@ -401,19 +428,28 @@ public static Properties generateProxyPropertiesForJDBC() {
       proxyProperties.put(
           SFSessionProperty.PROXY_PORT.getPropertyKey(), System.getProperty(PROXY_PORT));
 
+      LOGGER.info("Generated proxy properties - Host: {}, Port: {}", 
+                 System.getProperty(PROXY_HOST), System.getProperty(PROXY_PORT));
+
       // Check if proxy username and password are set
       final String proxyUser = System.getProperty(HTTP_PROXY_USER);
       final String proxyPassword = System.getProperty(HTTP_PROXY_PASSWORD);
       if (!isNullOrEmpty(proxyUser) && !isNullOrEmpty(proxyPassword)) {
         proxyProperties.put(SFSessionProperty.PROXY_USER.getPropertyKey(), proxyUser);
         proxyProperties.put(SFSessionProperty.PROXY_PASSWORD.getPropertyKey(), proxyPassword);
+        LOGGER.info("Added proxy authentication credentials to generated properties");
+      } else {
+        LOGGER.info("No proxy authentication credentials found in system properties");
       }
 
       // Check if http.nonProxyHosts was set
       final String nonProxyHosts = System.getProperty(NON_PROXY_HOSTS);
       if (!isNullOrEmpty(nonProxyHosts)) {
         proxyProperties.put(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey(), nonProxyHosts);
+        LOGGER.info("Added nonProxyHosts to generated properties: {}", nonProxyHosts);
       }
+    } else {
+      LOGGER.info("System property {} is not set to true, generating empty proxy properties", USE_PROXY);
     }
     return proxyProperties;
   }
@@ -496,15 +532,17 @@ public static void shutdownHttpConnectionManagerDaemonThread() {
    * Close all cached HTTP clients and clear the cache
    */
   public static void closeAllHttpClients() {
-    LOGGER.debug("Closing all cached HTTP clients");
+    LOGGER.info("Closing all cached HTTP clients. Total clients in cache: {}", httpClientCache.size());
     for (Map.Entry<HttpClientSettingsKey, CloseableHttpClient> entry : httpClientCache.entrySet()) {
       try {
+        LOGGER.debug("Closing HTTP client for key: {}", entry.getKey());
         entry.getValue().close();
       } catch (Exception e) {
         LOGGER.warn("Error closing HTTP client for key: {}", entry.getKey(), e);
       }
     }
     httpClientCache.clear();
+    LOGGER.info("All cached HTTP clients closed and cache cleared");
     shutdownHttpConnectionManagerDaemonThread();
   }
 
@@ -530,7 +568,15 @@ private static String createPoolStatsInfo(String title, PoolStats poolStats) {
    */
   public static Boolean shouldBypassProxy(String accountName) {
     String targetHost = accountName + SNOWFLAKE_DOMAIN_NAME;
-    return System.getProperty(NON_PROXY_HOSTS) != null && isInNonProxyHosts(targetHost);
+    boolean shouldBypass = System.getProperty(NON_PROXY_HOSTS) != null && isInNonProxyHosts(targetHost);
+    
+    if (shouldBypass) {
+      LOGGER.info("Account {} ({}) matches nonProxyHosts pattern. Bypassing proxy.", accountName, targetHost);
+    } else {
+      LOGGER.debug("Account {} ({}) does not match nonProxyHosts pattern or nonProxyHosts not set.", accountName, targetHost);
+    }
+    
+    return shouldBypass;
   }
 
   /**

From 111dacd86f5a1d088b3580041d62065bdf54bfbc Mon Sep 17 00:00:00 2001
From: Sangeet Mishra <mail2sangeetmishra@gmail.com>
Date: Sat, 14 Jun 2025 12:43:30 +0530
Subject: [PATCH 350/356] Reset new changes (#43)

* new changes

* Revert "new changes"

This reverts commit ed9d069f4b0be7dd90aeb4cbed5a65371dad5848.

* update semaphore.yml to use Ubuntu 24.04 for build agent

* set snowflake-ingest-sdk version to 2.1.2
---
 e2e-jar-test/pom.xml                                            | 2 +-
 pom.xml                                                         | 2 +-
 .../java/net/snowflake/ingest/connection/RequestBuilder.java    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/e2e-jar-test/pom.xml b/e2e-jar-test/pom.xml
index 8e7c72c51..bf319cf80 100644
--- a/e2e-jar-test/pom.xml
+++ b/e2e-jar-test/pom.xml
@@ -27,7 +27,7 @@
             <dependency>
                 <groupId>net.snowflake</groupId>
                 <artifactId>snowflake-ingest-sdk</artifactId>
-                <version>2.1.0-hotfix1</version>
+                <version>2.1.2</version>
             </dependency>
 
             <dependency>
diff --git a/pom.xml b/pom.xml
index 9f01aa0b6..281581c84 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
   <!-- Arifact name and version information -->
   <groupId>io.confluent</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.1.1</version>
+  <version>2.1.2</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index e44d82c14..822a64358 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.1.1";
+  public static final String DEFAULT_VERSION = "2.1.2";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 

From 44ded9ec26c672bc7807d7e88a5eb61dec028bd3 Mon Sep 17 00:00:00 2001
From: Sangeet Mishra <mail2sangeetmishra@gmail.com>
Date: Sat, 14 Jun 2025 13:19:13 +0530
Subject: [PATCH 351/356] reset files (#44)

---
 e2e-jar-test/pom.xml                          |   2 +-
 pom.xml                                       |   2 +-
 .../snowflake/ingest/SimpleIngestManager.java |  49 +----
 .../ingest/connection/RequestBuilder.java     |   2 +-
 .../streaming/internal/FlushService.java      |   3 +-
 ...nowflakeStreamingIngestClientInternal.java |  71 +------
 .../internal/StreamingIngestStage.java        |  27 +--
 .../net/snowflake/ingest/utils/HttpUtil.java  | 181 +++---------------
 .../net/snowflake/ingest/utils/Utils.java     |  20 +-
 9 files changed, 35 insertions(+), 322 deletions(-)

diff --git a/e2e-jar-test/pom.xml b/e2e-jar-test/pom.xml
index bf319cf80..9ab3e9ab0 100644
--- a/e2e-jar-test/pom.xml
+++ b/e2e-jar-test/pom.xml
@@ -27,7 +27,7 @@
             <dependency>
                 <groupId>net.snowflake</groupId>
                 <artifactId>snowflake-ingest-sdk</artifactId>
-                <version>2.1.2</version>
+                <version>2.1.3</version>
             </dependency>
 
             <dependency>
diff --git a/pom.xml b/pom.xml
index 281581c84..a49b61cd1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
   <!-- Arifact name and version information -->
   <groupId>io.confluent</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.1.2</version>
+  <version>2.1.3</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
index e6804af90..8df7e1727 100644
--- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
+++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
@@ -15,7 +15,6 @@
 import java.security.spec.InvalidKeySpecException;
 import java.util.Collections;
 import java.util.List;
-import java.util.Properties;
 import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
@@ -386,30 +385,6 @@ public SimpleIngestManager(
         new RequestBuilder(account, user, keyPair, schemeName, hostName, port, userAgentSuffix);
   }
 
-
-  /* Another flavor of constructor which supports userAgentSuffix and proxyProperties */
-  public SimpleIngestManager(
-    String account,
-    String user,
-    String pipe,
-    PrivateKey privateKey,
-    String schemeName,
-    String hostName,
-    int port,
-    String userAgentSuffix,
-    Properties proxyProperties)
-    throws NoSuchAlgorithmException, InvalidKeySpecException {
-  KeyPair keyPair = Utils.createKeyPairFromPrivateKey(privateKey);
-  // call our initializer method
-  init(account, user, pipe, keyPair, proxyProperties);
-
-  // make the request builder we'll use to build messages to the service
-  builder =
-      new RequestBuilder(account, user, keyPair, schemeName, hostName, port, userAgentSuffix);
-}
-
-  
-
   // ========= Constructors End =========
 
   /**
@@ -422,34 +397,14 @@ public SimpleIngestManager(
    * @param keyPair the KeyPair we'll use to sign JWT tokens
    */
   private void init(String account, String user, String pipe, KeyPair keyPair) {
-    init(account, user, pipe, keyPair, new Properties());
-  }
-
-  /**
-   * init - Does the basic work of constructing a SimpleIngestManager that is common across all
-   * constructors
-   *
-   * @param account The account into which we're loading
-   * @param user the user performing this load
-   * @param pipe the fully qualified name of pipe
-   * @param keyPair the KeyPair we'll use to sign JWT tokens
-   * @param proxyProperties proxy properties for HTTP client configuration
-   */
-  private void init(String account, String user, String pipe, KeyPair keyPair, Properties proxyProperties) {
     // set up our reference variables
     this.account = account;
     this.user = user;
     this.pipe = pipe;
     this.keyPair = keyPair;
 
-    // make our client for sending requests with proxy properties support
-    if (proxyProperties != null && !proxyProperties.isEmpty()) {
-      LOGGER.info("Creating HTTP client for SimpleIngestManager with proxy properties for account: {}, user: {}", account, user);
-    } else {
-      LOGGER.info("Creating HTTP client for SimpleIngestManager without proxy properties for account: {}, user: {}", account, user);
-    }
-    
-    httpClient = HttpUtil.getHttpClient(account, proxyProperties);
+    // make our client for sending requests
+    httpClient = HttpUtil.getHttpClient(account);
     // make the request builder we'll use to build messages to the service
   }
 
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 822a64358..08cd26a47 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.1.2";
+  public static final String DEFAULT_VERSION = "2.1.3";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 30dadbe66..2324adda8 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -167,8 +167,7 @@ List<List<ChannelData<T>>> getData() {
               client.getHttpClient(),
               client.getRequestBuilder(),
               client.getName(),
-              DEFAULT_MAX_UPLOAD_RETRIES,
-              client.getProxyProperties());
+              DEFAULT_MAX_UPLOAD_RETRIES);
     } catch (SnowflakeSQLException | IOException err) {
       throw new SFException(err, ErrorCode.UNABLE_TO_CONNECT_TO_STAGE);
     }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 51e6a4d86..62fc265ff 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -142,9 +142,6 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
   // Background thread that uploads telemetry data periodically
   private ScheduledExecutorService telemetryWorker;
 
-  // Store original properties for proxy configuration
-  private final Properties originalProperties;
-
   /**
    * Constructor
    *
@@ -165,33 +162,11 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
       RequestBuilder requestBuilder,
       Map<String, Object> parameterOverrides) {
     this.parameterProvider = new ParameterProvider(parameterOverrides, prop);
-    this.originalProperties = prop;
 
     this.name = name;
     String accountName = accountURL == null ? null : accountURL.getAccount();
     this.isTestMode = isTestMode;
-    
-    if (prop != null && !prop.isEmpty()) {
-      // Check if proxy-related properties are present
-      boolean hasProxyConfig = prop.stringPropertyNames().stream()
-          .anyMatch(key -> key.startsWith("http.proxy") || 
-                          key.equals("useProxy") || 
-                          key.equals("proxyHost") || 
-                          key.equals("proxyPort") ||
-                          key.equals("nonProxyHosts") ||
-                          key.equals("proxyUser") ||
-                          key.equals("proxyPassword"));
-      
-      if (hasProxyConfig) {
-        logger.logInfo("Creating HTTP client for SnowflakeStreamingIngestClient with proxy configuration for account: {}, client: {}", accountName, name);
-      } else {
-        logger.logInfo("Creating HTTP client for SnowflakeStreamingIngestClient without proxy configuration for account: {}, client: {}", accountName, name);
-      }
-    } else {
-      logger.logInfo("Creating HTTP client for SnowflakeStreamingIngestClient with no properties for account: {}, client: {}", accountName, name);
-    }
-    
-    this.httpClient = (httpClient != null) ? httpClient : HttpUtil.getHttpClient(accountName, prop);
+    this.httpClient = httpClient == null ? HttpUtil.getHttpClient(accountName) : httpClient;
     this.channelCache = new ChannelCache<>();
     this.isClosed = false;
     this.requestBuilder = requestBuilder;
@@ -1100,48 +1075,4 @@ private void cleanUpResources() {
       HttpUtil.shutdownHttpConnectionManagerDaemonThread();
     }
   }
-
-  /**
-   * Extract proxy properties from the original Properties object
-   *
-   * @return Properties containing proxy configuration
-   */
-  Properties getProxyProperties() {
-    Properties proxyProperties = new Properties();
-
-    
-    if (this.originalProperties != null) {
-
-      for (String key : this.originalProperties.stringPropertyNames()) {
-        if (key.equals(SFSessionProperty.USE_PROXY.getPropertyKey()) ||
-            key.equals(SFSessionProperty.PROXY_HOST.getPropertyKey()) ||
-            key.equals(SFSessionProperty.PROXY_PORT.getPropertyKey()) ||
-            key.equals(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey()) ||
-            key.equals(SFSessionProperty.PROXY_USER.getPropertyKey()) ||
-            key.equals(SFSessionProperty.PROXY_PASSWORD.getPropertyKey())) {
-          proxyProperties.put(key, this.originalProperties.getProperty(key));
-        }
-      }
-      
-      if (!proxyProperties.isEmpty()) {
-        logger.logInfo("Extracted {} proxy properties from original properties for client: {}", proxyProperties.size(), this.name);
-        logger.logDebug("Proxy properties extracted: {}", 
-                       proxyProperties.keySet().stream()
-                           .map(k -> k + "=" + (k.toString().toLowerCase().contains("password") ? "[HIDDEN]" : proxyProperties.get(k)))
-                           .collect(Collectors.joining(", ")));
-      } else {
-        logger.logInfo("No proxy properties found in original properties for client: {}", this.name);
-      }
-    } else {
-      logger.logInfo("Original properties is null, cannot extract proxy properties for client: {}", this.name);
-    }
-    
-    // If no proxy properties found in original properties, fall back to system properties
-    if (proxyProperties.isEmpty()) {
-      logger.logInfo("Falling back to system properties for proxy configuration for client: {}", this.name);
-      return HttpUtil.generateProxyPropertiesForJDBC();
-    }
-    
-    return proxyProperties;
-  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index e707d3b07..e8e56f383 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -98,34 +98,11 @@ state to record unknown age.
       String clientName,
       int maxUploadRetries)
       throws SnowflakeSQLException, IOException {
-    this(isTestMode, role, httpClient, requestBuilder, clientName, maxUploadRetries, null);
-  }
-
-  /**
-   * Constructor with proxy properties support
-   *
-   * @param isTestMode whether we're under test mode
-   * @param role Snowflake role used by the Client
-   * @param httpClient http client reference
-   * @param requestBuilder request builder to build the HTTP request
-   * @param clientName the client name
-   * @param maxUploadRetries maximum number of upload retries
-   * @param proxyProperties proxy properties for HTTP client configuration
-   */
-  StreamingIngestStage(
-      boolean isTestMode,
-      String role,
-      CloseableHttpClient httpClient,
-      RequestBuilder requestBuilder,
-      String clientName,
-      int maxUploadRetries,
-      Properties proxyProperties)
-      throws SnowflakeSQLException, IOException {
     this.httpClient = httpClient;
     this.role = role;
     this.requestBuilder = requestBuilder;
     this.clientName = clientName;
-    this.proxyProperties = proxyProperties != null ? proxyProperties : generateProxyPropertiesForJDBC();
+    this.proxyProperties = generateProxyPropertiesForJDBC();
     this.maxUploadRetries = maxUploadRetries;
 
     if (!isTestMode) {
@@ -152,7 +129,7 @@ state to record unknown age.
       SnowflakeFileTransferMetadataWithAge testMetadata,
       int maxRetryCount)
       throws SnowflakeSQLException, IOException {
-    this(isTestMode, role, httpClient, requestBuilder, clientName, maxRetryCount, null);
+    this(isTestMode, role, httpClient, requestBuilder, clientName, maxRetryCount);
     if (!isTestMode) {
       throw new SFException(ErrorCode.INTERNAL_ERROR);
     }
diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index 0c8769f85..1ff65a095 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -10,10 +10,8 @@
 import java.time.Duration;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
-import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.regex.Pattern;
@@ -68,8 +66,7 @@ public class HttpUtil {
    */
   private static final int MAX_RETRIES = 10;
 
-  private static final Map<HttpClientSettingsKey, CloseableHttpClient> httpClientCache =
-      new ConcurrentHashMap<>();
+  private static volatile CloseableHttpClient httpClient;
 
   private static PoolingHttpClientConnectionManager connectionManager;
 
@@ -103,100 +100,28 @@ public class HttpUtil {
   // Only connections that are currently owned, not checked out, are subject to idle timeouts.
   private static final int DEFAULT_IDLE_CONNECTION_TIMEOUT_SECONDS = 30;
 
-  private static final Logger LOGGER = LoggerFactory.getLogger(HttpUtil.class);
-
   /**
-   * Get HttpClient with proxy properties support
-   *
-   * @param accountName account name to connect to (excluding snowflakecomputing.com domain)
-   * @param proxyProperties proxy properties, can be null
+   * @param {@code String} account name to connect to (excluding snowflakecomputing.com domain)
    * @return Instance of CloseableHttpClient
    */
-  public static CloseableHttpClient getHttpClient(String accountName, Properties proxyProperties) {
-
-    HttpClientSettingsKey key = createHttpClientSettingsKey(accountName, proxyProperties);
-
-    CloseableHttpClient client = httpClientCache.get(key);
-    // todo: this is for testing phase
-    if (client != null) {
-      LOGGER.info("Reusing existing HTTP client for account: {}, key: {}", accountName, key);
-      return client;
+  public static CloseableHttpClient getHttpClient(String accountName) {
+    if (httpClient == null) {
+      synchronized (HttpUtil.class) {
+        if (httpClient == null) {
+          initHttpClient(accountName);
+        }
+      }
     }
 
-    LOGGER.info("No existing HTTP client found for account: {}, key: {}. Creating new HTTP client.", accountName, key);
-    return httpClientCache.computeIfAbsent(key, HttpUtil::buildHttpClient);
-  }
+    initIdleConnectionMonitoringThread();
 
-  /**
-   * @param accountName account name to connect to (excluding snowflakecomputing.com domain)
-   * @return Instance of CloseableHttpClient
-   */
-  public static CloseableHttpClient getHttpClient(String accountName) {
-    return getHttpClient(accountName, null);
+    return httpClient;
   }
 
-  /**
-   * Create HttpClientSettingsKey from account name and proxy properties
-   */
-  private static HttpClientSettingsKey createHttpClientSettingsKey(String accountName, Properties proxyProperties) {
-
-    if (proxyProperties != null && proxyProperties.containsKey(SFSessionProperty.USE_PROXY.getPropertyKey())) {
+  private static final Logger LOGGER = LoggerFactory.getLogger(HttpUtil.class);
 
-      Boolean useProxy = Boolean.valueOf(proxyProperties.getProperty(SFSessionProperty.USE_PROXY.getPropertyKey()));
+  private static void initHttpClient(String accountName) {
 
-      if (useProxy) {
-        String proxyHost = proxyProperties.getProperty(SFSessionProperty.PROXY_HOST.getPropertyKey(), "");
-        String proxyPortStr = proxyProperties.getProperty(SFSessionProperty.PROXY_PORT.getPropertyKey(), "0");
-        int proxyPort = 0;
-        try {
-          proxyPort = Integer.parseInt(proxyPortStr);
-        } catch (NumberFormatException e) {
-          throw new IllegalArgumentException(
-              "Invalid proxy port: '" + proxyPortStr + "'. Proxy port must be a valid integer.", e);
-        }
-        String nonProxyHosts = proxyProperties.getProperty(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey(), "");
-        String proxyUser = proxyProperties.getProperty(SFSessionProperty.PROXY_USER.getPropertyKey(), "");
-        String proxyPassword = proxyProperties.getProperty(SFSessionProperty.PROXY_PASSWORD.getPropertyKey(), "");
-        
-        LOGGER.info("Creating HTTP client settings key with proxy configuration from properties for account: {}. Proxy host: {}, port: {}, user: {}, nonProxyHosts: {}", 
-                   accountName, proxyHost, proxyPort, isNullOrEmpty(proxyUser) ? "not set" : "set", isNullOrEmpty(nonProxyHosts) ? "not set" : nonProxyHosts);
-        return new HttpClientSettingsKey(accountName, proxyHost, proxyPort, nonProxyHosts, proxyUser, proxyPassword);
-      } else {
-        LOGGER.info("Creating HTTP client settings key with proxy explicitly disabled via properties for account: {}", accountName);
-      }
-    }
-    
-    // Check system properties for proxy configuration (backward compatibility)
-    if ("true".equalsIgnoreCase(System.getProperty(USE_PROXY)) && !shouldBypassProxy(accountName)) {
-      String proxyHost = System.getProperty(PROXY_HOST, "");
-      String proxyPortStr = System.getProperty(PROXY_PORT, "0");
-      int proxyPort = 0;
-      try {
-        proxyPort = Integer.parseInt(proxyPortStr);
-      } catch (NumberFormatException e) {
-        throw new IllegalArgumentException(
-            "Invalid proxy port: '" + proxyPortStr + "'. Proxy port must be a valid integer.", e);
-      }
-      String nonProxyHosts = System.getProperty(NON_PROXY_HOSTS, "");
-      String proxyUser = System.getProperty(HTTP_PROXY_USER, "");
-      String proxyPassword = System.getProperty(HTTP_PROXY_PASSWORD, "");
-      
-      LOGGER.info("Creating HTTP client settings key with proxy configuration from system properties for account: {}. Proxy host: {}, port: {}, user: {}, nonProxyHosts: {}", 
-                 accountName, proxyHost, proxyPort, isNullOrEmpty(proxyUser) ? "not set" : "set", isNullOrEmpty(nonProxyHosts) ? "not set" : nonProxyHosts);
-      return new HttpClientSettingsKey(accountName, proxyHost, proxyPort, nonProxyHosts, proxyUser, proxyPassword);
-    }
-    
-    // No proxy configuration
-    LOGGER.info("Creating HTTP client settings key without proxy configuration for account: {}", accountName);
-    return new HttpClientSettingsKey(accountName);
-  }
-
-  /**
-   * Build HttpClient based on the settings key
-   */
-  private static CloseableHttpClient buildHttpClient(HttpClientSettingsKey key) {
-    LOGGER.info("Building new HTTP client for key: {}", key);
-    
     Security.setProperty("ocsp.enable", "true");
 
     SSLContext sslContext = SSLContexts.createDefault();
@@ -242,45 +167,34 @@ private static CloseableHttpClient buildHttpClient(HttpClientSettingsKey key) {
             .setDefaultRequestConfig(requestConfig);
 
     // proxy settings
-    if (key.usesProxy()) {
-      LOGGER.info("Configuring HTTP client with proxy settings. Host: {}, Port: {}", key.getProxyHost(), key.getProxyPort());
-      
-      if (isNullOrEmpty(key.getProxyHost())) {
+    if ("true".equalsIgnoreCase(System.getProperty(USE_PROXY)) && !shouldBypassProxy(accountName)) {
+      if (System.getProperty(PROXY_PORT) == null) {
         throw new IllegalArgumentException(
-            "proxy host IP is not provided, please assign proxy host IP to http.proxyHost option");
+            "proxy port number is not provided, please assign proxy port to http.proxyPort option");
       }
-      if (key.getProxyPort() <= 0) {
+      if (System.getProperty(PROXY_HOST) == null) {
         throw new IllegalArgumentException(
-            "proxy port number is not provided, please assign proxy port to http.proxyPort option");
+            "proxy host IP is not provided, please assign proxy host IP to http.proxyHost option");
       }
-      
-      String proxyHost = key.getProxyHost();
-      int proxyPort = key.getProxyPort();
+      String proxyHost = System.getProperty(PROXY_HOST);
+      int proxyPort = Integer.parseInt(System.getProperty(PROXY_PORT));
       HttpHost proxy = new HttpHost(proxyHost, proxyPort, PROXY_SCHEME);
       DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
       clientBuilder = clientBuilder.setRoutePlanner(routePlanner);
 
       // Check if proxy username and password are set
-      final String proxyUser = key.getProxyUser();
-      final String proxyPassword = key.getProxyPassword();
+      final String proxyUser = System.getProperty(HTTP_PROXY_USER);
+      final String proxyPassword = System.getProperty(HTTP_PROXY_PASSWORD);
       if (!isNullOrEmpty(proxyUser) && !isNullOrEmpty(proxyPassword)) {
-        LOGGER.info("Configuring HTTP client with proxy authentication credentials");
         Credentials credentials = new UsernamePasswordCredentials(proxyUser, proxyPassword);
         AuthScope authScope = new AuthScope(proxyHost, proxyPort);
         CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
         credentialsProvider.setCredentials(authScope, credentials);
         clientBuilder = clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
-      } else {
-        LOGGER.info("HTTP client configured with proxy but no authentication credentials provided");
       }
-    } else {
-      LOGGER.info("Configuring HTTP client without proxy settings");
     }
 
-    CloseableHttpClient httpClient = clientBuilder.build();
-    initIdleConnectionMonitoringThread();
-    LOGGER.info("Successfully built new HTTP client for key: {}", key);
-    return httpClient;
+    httpClient = clientBuilder.build();
   }
 
   /** Starts a daemon thread to monitor idle connections in http connection manager. */
@@ -410,8 +324,6 @@ private static String getRequestUriFromContext(final HttpContext httpContext) {
   public static Properties generateProxyPropertiesForJDBC() {
     Properties proxyProperties = new Properties();
     if (Boolean.parseBoolean(System.getProperty(USE_PROXY))) {
-      LOGGER.info("Generating proxy properties for JDBC from system properties");
-      
       if (isNullOrEmpty(System.getProperty(PROXY_PORT))) {
         throw new IllegalArgumentException(
             "proxy port number is not provided, please assign proxy port to http.proxyPort option");
@@ -428,28 +340,19 @@ public static Properties generateProxyPropertiesForJDBC() {
       proxyProperties.put(
           SFSessionProperty.PROXY_PORT.getPropertyKey(), System.getProperty(PROXY_PORT));
 
-      LOGGER.info("Generated proxy properties - Host: {}, Port: {}", 
-                 System.getProperty(PROXY_HOST), System.getProperty(PROXY_PORT));
-
       // Check if proxy username and password are set
       final String proxyUser = System.getProperty(HTTP_PROXY_USER);
       final String proxyPassword = System.getProperty(HTTP_PROXY_PASSWORD);
       if (!isNullOrEmpty(proxyUser) && !isNullOrEmpty(proxyPassword)) {
         proxyProperties.put(SFSessionProperty.PROXY_USER.getPropertyKey(), proxyUser);
         proxyProperties.put(SFSessionProperty.PROXY_PASSWORD.getPropertyKey(), proxyPassword);
-        LOGGER.info("Added proxy authentication credentials to generated properties");
-      } else {
-        LOGGER.info("No proxy authentication credentials found in system properties");
       }
 
       // Check if http.nonProxyHosts was set
       final String nonProxyHosts = System.getProperty(NON_PROXY_HOSTS);
       if (!isNullOrEmpty(nonProxyHosts)) {
         proxyProperties.put(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey(), nonProxyHosts);
-        LOGGER.info("Added nonProxyHosts to generated properties: {}", nonProxyHosts);
       }
-    } else {
-      LOGGER.info("System property {} is not set to true, generating empty proxy properties", USE_PROXY);
     }
     return proxyProperties;
   }
@@ -517,33 +420,7 @@ private boolean isShutdown() {
 
   /** Shuts down the daemon thread. */
   public static void shutdownHttpConnectionManagerDaemonThread() {
-    idleConnectionMonitorThreadLock.lock();
-    try {
-      if (idleConnectionMonitorThread != null) {
-        idleConnectionMonitorThread.shutdown();
-        idleConnectionMonitorThread = null;
-      }
-    } finally {
-      idleConnectionMonitorThreadLock.unlock();
-    }
-  }
-
-  /**
-   * Close all cached HTTP clients and clear the cache
-   */
-  public static void closeAllHttpClients() {
-    LOGGER.info("Closing all cached HTTP clients. Total clients in cache: {}", httpClientCache.size());
-    for (Map.Entry<HttpClientSettingsKey, CloseableHttpClient> entry : httpClientCache.entrySet()) {
-      try {
-        LOGGER.debug("Closing HTTP client for key: {}", entry.getKey());
-        entry.getValue().close();
-      } catch (Exception e) {
-        LOGGER.warn("Error closing HTTP client for key: {}", entry.getKey(), e);
-      }
-    }
-    httpClientCache.clear();
-    LOGGER.info("All cached HTTP clients closed and cache cleared");
-    shutdownHttpConnectionManagerDaemonThread();
+    idleConnectionMonitorThread.shutdown();
   }
 
   /** Create Pool stats for a route */
@@ -568,15 +445,7 @@ private static String createPoolStatsInfo(String title, PoolStats poolStats) {
    */
   public static Boolean shouldBypassProxy(String accountName) {
     String targetHost = accountName + SNOWFLAKE_DOMAIN_NAME;
-    boolean shouldBypass = System.getProperty(NON_PROXY_HOSTS) != null && isInNonProxyHosts(targetHost);
-    
-    if (shouldBypass) {
-      LOGGER.info("Account {} ({}) matches nonProxyHosts pattern. Bypassing proxy.", accountName, targetHost);
-    } else {
-      LOGGER.debug("Account {} ({}) does not match nonProxyHosts pattern or nonProxyHosts not set.", accountName, targetHost);
-    }
-    
-    return shouldBypass;
+    return System.getProperty(NON_PROXY_HOSTS) != null && isInNonProxyHosts(targetHost);
   }
 
   /**
diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java
index a9deb682b..a2653ab14 100644
--- a/src/main/java/net/snowflake/ingest/utils/Utils.java
+++ b/src/main/java/net/snowflake/ingest/utils/Utils.java
@@ -139,12 +139,7 @@ public static Properties createProperties(Properties inputProp) {
           privateKeyPassphrase = val;
           break;
         default:
-          // Preserve case for proxy-related properties that HttpUtil expects
-          if (isProxyRelatedProperty(key)) {
-            properties.put(key, val);
-          } else {
-            properties.put(key.toLowerCase(), val);
-          }
+          properties.put(key.toLowerCase(), val);
       }
     }
 
@@ -220,19 +215,6 @@ public static Properties createProperties(Properties inputProp) {
     return properties;
   }
 
-  /**
-   * Check if a property key is proxy-related and should preserve its case
-   */
-  private static boolean isProxyRelatedProperty(String key) {
-    return key.equals(SFSessionProperty.USE_PROXY.getPropertyKey()) ||
-           key.equals(SFSessionProperty.PROXY_HOST.getPropertyKey()) ||
-           key.equals(SFSessionProperty.PROXY_PORT.getPropertyKey()) ||
-           key.equals(SFSessionProperty.PROXY_USER.getPropertyKey()) ||
-           key.equals(SFSessionProperty.PROXY_PASSWORD.getPropertyKey()) ||
-           key.equals(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey()) ||
-           key.equals(SFSessionProperty.PROXY_PROTOCOL.getPropertyKey());
-  }
-
   /** Construct account url from input schema, host and port */
   public static String constructAccountUrl(String scheme, String host, int port) {
     return String.format("%s://%s:%d", scheme, host, port);

From 8e933aa08be747e0ec1b7fd9555c8a71f7ae2dc8 Mon Sep 17 00:00:00 2001
From: Sangeet Mishra <mail2sangeetmishra@gmail.com>
Date: Sat, 14 Jun 2025 13:36:16 +0530
Subject: [PATCH 352/356] bye bye HttpClientSettingsKey (#46)

* bye bye HttpClientSettingsKey

* bye bye cacheutiltest

* reset version
---
 e2e-jar-test/pom.xml                          |   2 +-
 pom.xml                                       |   2 +-
 .../ingest/connection/RequestBuilder.java     |   2 +-
 .../ingest/utils/HttpClientSettingsKey.java   | 125 ------------------
 .../ingest/utils/HttpUtilCacheTest.java       | 105 ---------------
 5 files changed, 3 insertions(+), 233 deletions(-)
 delete mode 100644 src/main/java/net/snowflake/ingest/utils/HttpClientSettingsKey.java
 delete mode 100644 src/test/java/net/snowflake/ingest/utils/HttpUtilCacheTest.java

diff --git a/e2e-jar-test/pom.xml b/e2e-jar-test/pom.xml
index 9ab3e9ab0..4ddff958e 100644
--- a/e2e-jar-test/pom.xml
+++ b/e2e-jar-test/pom.xml
@@ -27,7 +27,7 @@
             <dependency>
                 <groupId>net.snowflake</groupId>
                 <artifactId>snowflake-ingest-sdk</artifactId>
-                <version>2.1.3</version>
+                <version>2.1.4</version>
             </dependency>
 
             <dependency>
diff --git a/pom.xml b/pom.xml
index a49b61cd1..0d896116a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
   <!-- Arifact name and version information -->
   <groupId>io.confluent</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.1.3</version>
+  <version>2.1.4</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 08cd26a47..8631e420c 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.1.3";
+  public static final String DEFAULT_VERSION = "2.1.4";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 
diff --git a/src/main/java/net/snowflake/ingest/utils/HttpClientSettingsKey.java b/src/main/java/net/snowflake/ingest/utils/HttpClientSettingsKey.java
deleted file mode 100644
index b089d81f2..000000000
--- a/src/main/java/net/snowflake/ingest/utils/HttpClientSettingsKey.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Copyright (c) 2012-2017 Snowflake Computing Inc. All rights reserved.
- */
-
-package net.snowflake.ingest.utils;
-
-import static net.snowflake.ingest.utils.Utils.isNullOrEmpty;
-
-import java.io.Serializable;
-import java.util.Objects;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class defines all parameters needed to create an HttpClient object for the ingest service.
- * It is used as the key for the static hashmap of reusable http clients.
- */
-public class HttpClientSettingsKey implements Serializable {
-
-  private static final Logger LOGGER = LoggerFactory.getLogger(HttpClientSettingsKey.class);
-
-  private boolean useProxy;
-  private String proxyHost = "";
-  private int proxyPort = 0;
-  private String nonProxyHosts = "";
-  private String proxyUser = "";
-  private String proxyPassword = "";
-  private String accountName = "";
-
-  /**
-   * Constructor for proxy configuration
-   */
-  public HttpClientSettingsKey(
-      String accountName,
-      String proxyHost,
-      int proxyPort,
-      String nonProxyHosts,
-      String proxyUser,
-      String proxyPassword) {
-    this.useProxy = true;
-    this.accountName = !isNullOrEmpty(accountName) ? accountName.trim() : "";
-    this.proxyHost = !isNullOrEmpty(proxyHost) ? proxyHost.trim() : "";
-    this.proxyPort = proxyPort;
-    this.nonProxyHosts = !isNullOrEmpty(nonProxyHosts) ? nonProxyHosts.trim() : "";
-    this.proxyUser = !isNullOrEmpty(proxyUser) ? proxyUser.trim() : "";
-    this.proxyPassword = !isNullOrEmpty(proxyPassword) ? proxyPassword.trim() : "";
-    
-    LOGGER.debug("Created HttpClientSettingsKey with proxy configuration for account: {}. Host: {}, Port: {}, User: {}, NonProxyHosts: {}", 
-                this.accountName, this.proxyHost, this.proxyPort, 
-                !isNullOrEmpty(this.proxyUser) ? "set" : "not set", 
-                !isNullOrEmpty(this.nonProxyHosts) ? this.nonProxyHosts : "not set");
-  }
-
-  /**
-   * Constructor for non-proxy configuration
-   */
-  public HttpClientSettingsKey(String accountName) {
-    this.useProxy = false;
-    this.accountName = !isNullOrEmpty(accountName) ? accountName.trim() : "";
-    
-    LOGGER.debug("Created HttpClientSettingsKey without proxy configuration for account: {}", this.accountName);
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    if (this == obj) return true;
-    if (obj == null || getClass() != obj.getClass()) return false;
-    
-    HttpClientSettingsKey that = (HttpClientSettingsKey) obj;
-    
-    return useProxy == that.useProxy &&
-           proxyPort == that.proxyPort &&
-           Objects.equals(accountName, that.accountName) &&
-           Objects.equals(proxyHost, that.proxyHost) &&
-           Objects.equals(nonProxyHosts, that.nonProxyHosts) &&
-           Objects.equals(proxyUser, that.proxyUser) &&
-           Objects.equals(proxyPassword, that.proxyPassword);
-  }
-
-  @Override
-  public int hashCode() {
-    return Objects.hash(useProxy, accountName, proxyHost, proxyPort, nonProxyHosts, proxyUser, proxyPassword);
-  }
-
-  public boolean usesProxy() {
-    return this.useProxy;
-  }
-
-  public String getAccountName() {
-    return this.accountName;
-  }
-
-  public String getProxyHost() {
-    return this.proxyHost;
-  }
-
-  public int getProxyPort() {
-    return this.proxyPort;
-  }
-
-  public String getProxyUser() {
-    return this.proxyUser;
-  }
-
-  public String getProxyPassword() {
-    return this.proxyPassword;
-  }
-
-  public String getNonProxyHosts() {
-    return this.nonProxyHosts;
-  }
-
-  @Override
-  public String toString() {
-    return "HttpClientSettingsKey[" +
-           "accountName='" + accountName + '\'' +
-           ", useProxy=" + useProxy +
-           ", proxyHost='" + proxyHost + '\'' +
-           ", proxyPort=" + proxyPort +
-           ", nonProxyHosts='" + nonProxyHosts + '\'' +
-           ", proxyUser='" + proxyUser + '\'' +
-           ", proxyPassword=" + (proxyPassword.isEmpty() ? "not set" : "set") +
-           ']';
-  }
-} 
diff --git a/src/test/java/net/snowflake/ingest/utils/HttpUtilCacheTest.java b/src/test/java/net/snowflake/ingest/utils/HttpUtilCacheTest.java
deleted file mode 100644
index 167930dce..000000000
--- a/src/test/java/net/snowflake/ingest/utils/HttpUtilCacheTest.java
+++ /dev/null
@@ -1,105 +0,0 @@
-package net.snowflake.ingest.utils;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertSame;
-
-import java.util.Properties;
-import net.snowflake.client.core.SFSessionProperty;
-import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
-import org.junit.After;
-import org.junit.Test;
-
-/**
- * Test class for HttpUtil caching functionality.
- * Tests verify that HTTP clients are properly cached based on account name and proxy settings.
- * Note: Both JDBC and streaming clients now use unified SFSessionProperty constants for proxy configuration.
- */
-public class HttpUtilCacheTest {
-
-  @After
-  public void tearDown() {
-    // Clean up after each test
-    HttpUtil.closeAllHttpClients();
-  }
-
-  @Test
-  public void testHttpClientCachingWithSameAccount() {
-    String accountName = "testaccount";
-    
-    // Get HTTP client twice with same account
-    CloseableHttpClient client1 = HttpUtil.getHttpClient(accountName);
-    CloseableHttpClient client2 = HttpUtil.getHttpClient(accountName);
-    
-    // Should return the same instance
-    assertSame("HTTP clients should be the same for same account", client1, client2);
-  }
-
-  @Test
-  public void testHttpClientCachingWithDifferentAccounts() {
-    String accountName1 = "testaccount1";
-    String accountName2 = "testaccount2";
-    
-    // Get HTTP clients for different accounts
-    CloseableHttpClient client1 = HttpUtil.getHttpClient(accountName1);
-    CloseableHttpClient client2 = HttpUtil.getHttpClient(accountName2);
-    
-    // Should return different instances
-    assertNotSame("HTTP clients should be different for different accounts", client1, client2);
-  }
-
-  @Test
-  public void testHttpClientCachingWithSameProxySettings() {
-    String accountName = "testaccount";
-    Properties proxyProps = new Properties();
-    proxyProps.setProperty(SFSessionProperty.USE_PROXY.getPropertyKey(), "true");
-    proxyProps.setProperty(SFSessionProperty.PROXY_HOST.getPropertyKey(), "proxy.example.com");
-    proxyProps.setProperty(SFSessionProperty.PROXY_PORT.getPropertyKey(), "8080");
-    
-    // Get HTTP client twice with same proxy settings
-    CloseableHttpClient client1 = HttpUtil.getHttpClient(accountName, proxyProps);
-    CloseableHttpClient client2 = HttpUtil.getHttpClient(accountName, proxyProps);
-    
-    // Should return the same instance
-    assertSame("HTTP clients should be the same for same proxy settings", client1, client2);
-  }
-
-  @Test
-  public void testHttpClientCachingWithDifferentProxySettings() {
-    String accountName = "testaccount";
-    
-    Properties proxyProps1 = new Properties();
-    proxyProps1.setProperty(SFSessionProperty.USE_PROXY.getPropertyKey(), "true");
-    proxyProps1.setProperty(SFSessionProperty.PROXY_HOST.getPropertyKey(), "proxy1.example.com");
-    proxyProps1.setProperty(SFSessionProperty.PROXY_PORT.getPropertyKey(), "8080");
-    
-    Properties proxyProps2 = new Properties();
-    proxyProps2.setProperty(SFSessionProperty.USE_PROXY.getPropertyKey(), "true");
-    proxyProps2.setProperty(SFSessionProperty.PROXY_HOST.getPropertyKey(), "proxy2.example.com");
-    proxyProps2.setProperty(SFSessionProperty.PROXY_PORT.getPropertyKey(), "8080");
-    
-    // Get HTTP clients with different proxy settings
-    CloseableHttpClient client1 = HttpUtil.getHttpClient(accountName, proxyProps1);
-    CloseableHttpClient client2 = HttpUtil.getHttpClient(accountName, proxyProps2);
-    
-    // Should return different instances
-    assertNotSame("HTTP clients should be different for different proxy settings", client1, client2);
-  }
-
-  @Test
-  public void testHttpClientCachingWithProxyAndNoProxy() {
-    String accountName = "testaccount";
-    
-    Properties proxyProps = new Properties();
-    proxyProps.setProperty(SFSessionProperty.USE_PROXY.getPropertyKey(), "true");
-    proxyProps.setProperty(SFSessionProperty.PROXY_HOST.getPropertyKey(), "proxy.example.com");
-    proxyProps.setProperty(SFSessionProperty.PROXY_PORT.getPropertyKey(), "8080");
-    
-    // Get HTTP client with proxy and without proxy
-    CloseableHttpClient clientWithProxy = HttpUtil.getHttpClient(accountName, proxyProps);
-    CloseableHttpClient clientWithoutProxy = HttpUtil.getHttpClient(accountName);
-    
-    // Should return different instances
-    assertNotSame("HTTP clients should be different for proxy vs no proxy", clientWithProxy, clientWithoutProxy);
-  }
-} 

From f92355c11ec379c1c2451dee1f8f8b26d31b2510 Mon Sep 17 00:00:00 2001
From: Sangeet Mishra <mail2sangeetmishra@gmail.com>
Date: Tue, 17 Jun 2025 15:12:43 +0530
Subject: [PATCH 353/356] CC-33579 - introduce connector level proxy configs
 and make HttpClient keyed on proxy [re-raise] (#45)

* Add HttpClientSettingsKey class and corresponding tests for caching functionality

* Enhance proxy bypass logic to support properties and system settings

* fix test

* change log levels

* ./format.sh with temurin 11
---
 e2e-jar-test/pom.xml                          |   2 +-
 pom.xml                                       |   2 +-
 .../snowflake/ingest/SimpleIngestManager.java |  55 +++-
 .../ingest/connection/RequestBuilder.java     |   2 +-
 .../streaming/internal/FlushService.java      |   3 +-
 ...nowflakeStreamingIngestClientInternal.java |  94 +++++-
 .../internal/StreamingIngestStage.java        |  28 +-
 .../ingest/utils/HttpClientSettingsKey.java   | 139 +++++++++
 .../net/snowflake/ingest/utils/HttpUtil.java  | 267 +++++++++++++++---
 .../net/snowflake/ingest/utils/Utils.java     |  18 +-
 .../ingest/utils/HttpUtilCacheTest.java       | 108 +++++++
 11 files changed, 675 insertions(+), 43 deletions(-)
 create mode 100644 src/main/java/net/snowflake/ingest/utils/HttpClientSettingsKey.java
 create mode 100644 src/test/java/net/snowflake/ingest/utils/HttpUtilCacheTest.java

diff --git a/e2e-jar-test/pom.xml b/e2e-jar-test/pom.xml
index 4ddff958e..9a6933654 100644
--- a/e2e-jar-test/pom.xml
+++ b/e2e-jar-test/pom.xml
@@ -27,7 +27,7 @@
             <dependency>
                 <groupId>net.snowflake</groupId>
                 <artifactId>snowflake-ingest-sdk</artifactId>
-                <version>2.1.4</version>
+                <version>2.1.5</version>
             </dependency>
 
             <dependency>
diff --git a/pom.xml b/pom.xml
index 0d896116a..ec3f345d4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
   <!-- Arifact name and version information -->
   <groupId>io.confluent</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.1.4</version>
+  <version>2.1.5</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
index 8df7e1727..3a5fe71ed 100644
--- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
+++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java
@@ -15,6 +15,7 @@
 import java.security.spec.InvalidKeySpecException;
 import java.util.Collections;
 import java.util.List;
+import java.util.Properties;
 import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
@@ -385,6 +386,27 @@ public SimpleIngestManager(
         new RequestBuilder(account, user, keyPair, schemeName, hostName, port, userAgentSuffix);
   }
 
+  /* Another flavor of constructor which supports userAgentSuffix and proxyProperties */
+  public SimpleIngestManager(
+      String account,
+      String user,
+      String pipe,
+      PrivateKey privateKey,
+      String schemeName,
+      String hostName,
+      int port,
+      String userAgentSuffix,
+      Properties proxyProperties)
+      throws NoSuchAlgorithmException, InvalidKeySpecException {
+    KeyPair keyPair = Utils.createKeyPairFromPrivateKey(privateKey);
+    // call our initializer method
+    init(account, user, pipe, keyPair, proxyProperties);
+
+    // make the request builder we'll use to build messages to the service
+    builder =
+        new RequestBuilder(account, user, keyPair, schemeName, hostName, port, userAgentSuffix);
+  }
+
   // ========= Constructors End =========
 
   /**
@@ -397,14 +419,43 @@ public SimpleIngestManager(
    * @param keyPair the KeyPair we'll use to sign JWT tokens
    */
   private void init(String account, String user, String pipe, KeyPair keyPair) {
+    init(account, user, pipe, keyPair, new Properties());
+  }
+
+  /**
+   * init - Does the basic work of constructing a SimpleIngestManager that is common across all
+   * constructors
+   *
+   * @param account The account into which we're loading
+   * @param user the user performing this load
+   * @param pipe the fully qualified name of pipe
+   * @param keyPair the KeyPair we'll use to sign JWT tokens
+   * @param proxyProperties proxy properties for HTTP client configuration
+   */
+  private void init(
+      String account, String user, String pipe, KeyPair keyPair, Properties proxyProperties) {
     // set up our reference variables
     this.account = account;
     this.user = user;
     this.pipe = pipe;
     this.keyPair = keyPair;
 
-    // make our client for sending requests
-    httpClient = HttpUtil.getHttpClient(account);
+    // make our client for sending requests with proxy properties support
+    if (proxyProperties != null && !proxyProperties.isEmpty()) {
+      LOGGER.debug(
+          "Creating HTTP client for SimpleIngestManager with proxy properties for account: {},"
+              + " user: {}",
+          account,
+          user);
+    } else {
+      LOGGER.debug(
+          "Creating HTTP client for SimpleIngestManager without proxy properties for account: {},"
+              + " user: {}",
+          account,
+          user);
+    }
+
+    httpClient = HttpUtil.getHttpClient(account, proxyProperties);
     // make the request builder we'll use to build messages to the service
   }
 
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index 8631e420c..affee8161 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.1.4";
+  public static final String DEFAULT_VERSION = "2.1.5";
 
   public static final String JAVA_USER_AGENT = "JAVA";
 
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
index 2324adda8..30dadbe66 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java
@@ -167,7 +167,8 @@ List<List<ChannelData<T>>> getData() {
               client.getHttpClient(),
               client.getRequestBuilder(),
               client.getName(),
-              DEFAULT_MAX_UPLOAD_RETRIES);
+              DEFAULT_MAX_UPLOAD_RETRIES,
+              client.getProxyProperties());
     } catch (SnowflakeSQLException | IOException err) {
       throw new SFException(err, ErrorCode.UNABLE_TO_CONNECT_TO_STAGE);
     }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
index 62fc265ff..c23df3e34 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java
@@ -142,6 +142,9 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
   // Background thread that uploads telemetry data periodically
   private ScheduledExecutorService telemetryWorker;
 
+  // Store original properties for proxy configuration
+  private final Properties originalProperties;
+
   /**
    * Constructor
    *
@@ -162,11 +165,48 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
       RequestBuilder requestBuilder,
       Map<String, Object> parameterOverrides) {
     this.parameterProvider = new ParameterProvider(parameterOverrides, prop);
+    this.originalProperties = prop;
 
     this.name = name;
     String accountName = accountURL == null ? null : accountURL.getAccount();
     this.isTestMode = isTestMode;
-    this.httpClient = httpClient == null ? HttpUtil.getHttpClient(accountName) : httpClient;
+
+    if (prop != null && !prop.isEmpty()) {
+      // Check if proxy-related properties are present
+      boolean hasProxyConfig =
+          prop.stringPropertyNames().stream()
+              .anyMatch(
+                  key ->
+                      key.startsWith("http.proxy")
+                          || key.equals("useProxy")
+                          || key.equals("proxyHost")
+                          || key.equals("proxyPort")
+                          || key.equals("nonProxyHosts")
+                          || key.equals("proxyUser")
+                          || key.equals("proxyPassword"));
+
+      if (hasProxyConfig) {
+        logger.logDebug(
+            "Creating HTTP client for SnowflakeStreamingIngestClient with proxy configuration for"
+                + " account: {}, client: {}",
+            accountName,
+            name);
+      } else {
+        logger.logDebug(
+            "Creating HTTP client for SnowflakeStreamingIngestClient without proxy configuration"
+                + " for account: {}, client: {}",
+            accountName,
+            name);
+      }
+    } else {
+      logger.logDebug(
+          "Creating HTTP client for SnowflakeStreamingIngestClient with no properties for account:"
+              + " {}, client: {}",
+          accountName,
+          name);
+    }
+
+    this.httpClient = (httpClient != null) ? httpClient : HttpUtil.getHttpClient(accountName, prop);
     this.channelCache = new ChannelCache<>();
     this.isClosed = false;
     this.requestBuilder = requestBuilder;
@@ -1075,4 +1115,56 @@ private void cleanUpResources() {
       HttpUtil.shutdownHttpConnectionManagerDaemonThread();
     }
   }
+
+  /**
+   * Extract proxy properties from the original Properties object
+   *
+   * @return Properties containing proxy configuration
+   */
+  Properties getProxyProperties() {
+    Properties proxyProperties = new Properties();
+
+    if (this.originalProperties != null) {
+
+      for (String key : this.originalProperties.stringPropertyNames()) {
+        if (key.equals(SFSessionProperty.USE_PROXY.getPropertyKey())
+            || key.equals(SFSessionProperty.PROXY_HOST.getPropertyKey())
+            || key.equals(SFSessionProperty.PROXY_PORT.getPropertyKey())
+            || key.equals(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey())
+            || key.equals(SFSessionProperty.PROXY_USER.getPropertyKey())
+            || key.equals(SFSessionProperty.PROXY_PASSWORD.getPropertyKey())) {
+          proxyProperties.put(key, this.originalProperties.getProperty(key));
+        }
+      }
+
+      if (!proxyProperties.isEmpty()) {
+        logger.logTrace(
+            "Proxy properties extracted: {}",
+            proxyProperties.keySet().stream()
+                .map(
+                    k ->
+                        k
+                            + "="
+                            + (k.toString().toLowerCase().contains("password")
+                                ? "[HIDDEN]"
+                                : proxyProperties.get(k)))
+                .collect(Collectors.joining(", ")));
+      } else {
+        logger.logDebug(
+            "No proxy properties found in original properties for client: {}", this.name);
+      }
+    } else {
+      logger.logDebug(
+          "Original properties is null, cannot extract proxy properties for client: {}", this.name);
+    }
+
+    // If no proxy properties found in original properties, fall back to system properties
+    if (proxyProperties.isEmpty()) {
+      logger.logDebug(
+          "Falling back to system properties for proxy configuration for client: {}", this.name);
+      return HttpUtil.generateProxyPropertiesForJDBC();
+    }
+
+    return proxyProperties;
+  }
 }
diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
index e8e56f383..bd1ca79c7 100644
--- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
+++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java
@@ -98,11 +98,35 @@ state to record unknown age.
       String clientName,
       int maxUploadRetries)
       throws SnowflakeSQLException, IOException {
+    this(isTestMode, role, httpClient, requestBuilder, clientName, maxUploadRetries, null);
+  }
+
+  /**
+   * Constructor with proxy properties support
+   *
+   * @param isTestMode whether we're under test mode
+   * @param role Snowflake role used by the Client
+   * @param httpClient http client reference
+   * @param requestBuilder request builder to build the HTTP request
+   * @param clientName the client name
+   * @param maxUploadRetries maximum number of upload retries
+   * @param proxyProperties proxy properties for HTTP client configuration
+   */
+  StreamingIngestStage(
+      boolean isTestMode,
+      String role,
+      CloseableHttpClient httpClient,
+      RequestBuilder requestBuilder,
+      String clientName,
+      int maxUploadRetries,
+      Properties proxyProperties)
+      throws SnowflakeSQLException, IOException {
     this.httpClient = httpClient;
     this.role = role;
     this.requestBuilder = requestBuilder;
     this.clientName = clientName;
-    this.proxyProperties = generateProxyPropertiesForJDBC();
+    this.proxyProperties =
+        proxyProperties != null ? proxyProperties : generateProxyPropertiesForJDBC();
     this.maxUploadRetries = maxUploadRetries;
 
     if (!isTestMode) {
@@ -129,7 +153,7 @@ state to record unknown age.
       SnowflakeFileTransferMetadataWithAge testMetadata,
       int maxRetryCount)
       throws SnowflakeSQLException, IOException {
-    this(isTestMode, role, httpClient, requestBuilder, clientName, maxRetryCount);
+    this(isTestMode, role, httpClient, requestBuilder, clientName, maxRetryCount, null);
     if (!isTestMode) {
       throw new SFException(ErrorCode.INTERNAL_ERROR);
     }
diff --git a/src/main/java/net/snowflake/ingest/utils/HttpClientSettingsKey.java b/src/main/java/net/snowflake/ingest/utils/HttpClientSettingsKey.java
new file mode 100644
index 000000000..d25641980
--- /dev/null
+++ b/src/main/java/net/snowflake/ingest/utils/HttpClientSettingsKey.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) 2012-2017 Snowflake Computing Inc. All rights reserved.
+ */
+
+package net.snowflake.ingest.utils;
+
+import static net.snowflake.ingest.utils.Utils.isNullOrEmpty;
+
+import java.io.Serializable;
+import java.util.Objects;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class defines all parameters needed to create an HttpClient object for the ingest service.
+ * It is used as the key for the static hashmap of reusable http clients.
+ */
+public class HttpClientSettingsKey implements Serializable {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(HttpClientSettingsKey.class);
+
+  private boolean useProxy;
+  private String proxyHost = "";
+  private int proxyPort = 0;
+  private String nonProxyHosts = "";
+  private String proxyUser = "";
+  private String proxyPassword = "";
+  private String accountName = "";
+
+  /** Constructor for proxy configuration */
+  public HttpClientSettingsKey(
+      String accountName,
+      String proxyHost,
+      int proxyPort,
+      String nonProxyHosts,
+      String proxyUser,
+      String proxyPassword) {
+    this.useProxy = true;
+    this.accountName = !isNullOrEmpty(accountName) ? accountName.trim() : "";
+    this.proxyHost = !isNullOrEmpty(proxyHost) ? proxyHost.trim() : "";
+    this.proxyPort = proxyPort;
+    this.nonProxyHosts = !isNullOrEmpty(nonProxyHosts) ? nonProxyHosts.trim() : "";
+    this.proxyUser = !isNullOrEmpty(proxyUser) ? proxyUser.trim() : "";
+    this.proxyPassword = !isNullOrEmpty(proxyPassword) ? proxyPassword.trim() : "";
+
+    LOGGER.trace(
+        "Created HttpClientSettingsKey with proxy configuration for account: {}. Host: {}, Port:"
+            + " {}, User: {}, NonProxyHosts: {}",
+        this.accountName,
+        this.proxyHost,
+        this.proxyPort,
+        !isNullOrEmpty(this.proxyUser) ? "set" : "not set",
+        !isNullOrEmpty(this.nonProxyHosts) ? this.nonProxyHosts : "not set");
+  }
+
+  /** Constructor for non-proxy configuration */
+  public HttpClientSettingsKey(String accountName) {
+    this.useProxy = false;
+    this.accountName = !isNullOrEmpty(accountName) ? accountName.trim() : "";
+
+    LOGGER.debug(
+        "Created HttpClientSettingsKey without proxy configuration for account: {}",
+        this.accountName);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) return true;
+    if (obj == null || getClass() != obj.getClass()) return false;
+
+    HttpClientSettingsKey that = (HttpClientSettingsKey) obj;
+
+    return useProxy == that.useProxy
+        && proxyPort == that.proxyPort
+        && Objects.equals(accountName, that.accountName)
+        && Objects.equals(proxyHost, that.proxyHost)
+        && Objects.equals(nonProxyHosts, that.nonProxyHosts)
+        && Objects.equals(proxyUser, that.proxyUser)
+        && Objects.equals(proxyPassword, that.proxyPassword);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(
+        useProxy, accountName, proxyHost, proxyPort, nonProxyHosts, proxyUser, proxyPassword);
+  }
+
+  public boolean usesProxy() {
+    return this.useProxy;
+  }
+
+  public String getAccountName() {
+    return this.accountName;
+  }
+
+  public String getProxyHost() {
+    return this.proxyHost;
+  }
+
+  public int getProxyPort() {
+    return this.proxyPort;
+  }
+
+  public String getProxyUser() {
+    return this.proxyUser;
+  }
+
+  public String getProxyPassword() {
+    return this.proxyPassword;
+  }
+
+  public String getNonProxyHosts() {
+    return this.nonProxyHosts;
+  }
+
+  @Override
+  public String toString() {
+    return "HttpClientSettingsKey["
+        + "accountName='"
+        + accountName
+        + '\''
+        + ", useProxy="
+        + useProxy
+        + ", proxyHost='"
+        + proxyHost
+        + '\''
+        + ", proxyPort="
+        + proxyPort
+        + ", nonProxyHosts='"
+        + nonProxyHosts
+        + '\''
+        + ", proxyUser='"
+        + proxyUser
+        + '\''
+        + ", proxyPassword="
+        + (proxyPassword.isEmpty() ? "not set" : "set")
+        + ']';
+  }
+}
diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
index 1ff65a095..ab11ea7ee 100644
--- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
+++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java
@@ -10,8 +10,10 @@
 import java.time.Duration;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.regex.Pattern;
@@ -66,7 +68,8 @@ public class HttpUtil {
    */
   private static final int MAX_RETRIES = 10;
 
-  private static volatile CloseableHttpClient httpClient;
+  private static final Map<HttpClientSettingsKey, CloseableHttpClient> httpClientCache =
+      new ConcurrentHashMap<>();
 
   private static PoolingHttpClientConnectionManager connectionManager;
 
@@ -100,27 +103,135 @@ public class HttpUtil {
   // Only connections that are currently owned, not checked out, are subject to idle timeouts.
   private static final int DEFAULT_IDLE_CONNECTION_TIMEOUT_SECONDS = 30;
 
+  private static final Logger LOGGER = LoggerFactory.getLogger(HttpUtil.class);
+
+  /**
+   * Get HttpClient with proxy properties support
+   *
+   * @param accountName account name to connect to (excluding snowflakecomputing.com domain)
+   * @param proxyProperties proxy properties, can be null
+   * @return Instance of CloseableHttpClient
+   */
+  public static CloseableHttpClient getHttpClient(String accountName, Properties proxyProperties) {
+
+    HttpClientSettingsKey key = createHttpClientSettingsKey(accountName, proxyProperties);
+
+    CloseableHttpClient client = httpClientCache.get(key);
+    // todo: this is for testing phase
+    if (client != null) {
+      LOGGER.debug("Reusing existing HTTP client for account: {}, key: {}", accountName, key);
+      return client;
+    }
+
+    LOGGER.debug(
+        "No existing HTTP client found for account: {}, key: {}. Creating new HTTP client.",
+        accountName,
+        key);
+    return httpClientCache.computeIfAbsent(key, HttpUtil::buildHttpClient);
+  }
+
   /**
-   * @param {@code String} account name to connect to (excluding snowflakecomputing.com domain)
+   * @param accountName account name to connect to (excluding snowflakecomputing.com domain)
    * @return Instance of CloseableHttpClient
    */
   public static CloseableHttpClient getHttpClient(String accountName) {
-    if (httpClient == null) {
-      synchronized (HttpUtil.class) {
-        if (httpClient == null) {
-          initHttpClient(accountName);
+    return getHttpClient(accountName, null);
+  }
+
+  /** Create HttpClientSettingsKey from account name and proxy properties */
+  private static HttpClientSettingsKey createHttpClientSettingsKey(
+      String accountName, Properties proxyProperties) {
+
+    if (proxyProperties != null
+        && proxyProperties.containsKey(SFSessionProperty.USE_PROXY.getPropertyKey())) {
+
+      Boolean useProxy =
+          Boolean.valueOf(
+              proxyProperties.getProperty(SFSessionProperty.USE_PROXY.getPropertyKey()));
+
+      if (useProxy) {
+        String proxyHost =
+            proxyProperties.getProperty(SFSessionProperty.PROXY_HOST.getPropertyKey(), "");
+        String proxyPortStr =
+            proxyProperties.getProperty(SFSessionProperty.PROXY_PORT.getPropertyKey(), "0");
+        int proxyPort = 0;
+        try {
+          proxyPort = Integer.parseInt(proxyPortStr);
+        } catch (NumberFormatException e) {
+          throw new IllegalArgumentException(
+              "Invalid proxy port: '" + proxyPortStr + "'. Proxy port must be a valid integer.", e);
         }
+        String nonProxyHosts =
+            proxyProperties.getProperty(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey(), "");
+        String proxyUser =
+            proxyProperties.getProperty(SFSessionProperty.PROXY_USER.getPropertyKey(), "");
+        String proxyPassword =
+            proxyProperties.getProperty(SFSessionProperty.PROXY_PASSWORD.getPropertyKey(), "");
+
+        if (shouldBypassProxy(accountName, proxyProperties)) {
+          LOGGER.debug(
+              "Account {} matches nonProxyHosts pattern from system properties. Bypassing proxy"
+                  + " despite proxyProperties configuration.",
+              accountName);
+          return new HttpClientSettingsKey(accountName);
+        }
+
+        LOGGER.trace(
+            "Creating HTTP client settings key with proxy configuration from properties for"
+                + " account: {}. Proxy host: {}, port: {}, user: {}, nonProxyHosts: {}",
+            accountName,
+            proxyHost,
+            proxyPort,
+            isNullOrEmpty(proxyUser) ? "not set" : "set",
+            isNullOrEmpty(nonProxyHosts) ? "not set" : nonProxyHosts);
+        return new HttpClientSettingsKey(
+            accountName, proxyHost, proxyPort, nonProxyHosts, proxyUser, proxyPassword);
+      } else {
+        LOGGER.debug(
+            "Creating HTTP client settings key with proxy explicitly disabled via properties for"
+                + " account: {}",
+            accountName);
       }
     }
 
-    initIdleConnectionMonitoringThread();
+    // Check system properties for proxy configuration (backward compatibility)
+    if ("true".equalsIgnoreCase(System.getProperty(USE_PROXY))
+        && !shouldBypassProxy(accountName, proxyProperties)) {
+      String proxyHost = System.getProperty(PROXY_HOST, "");
+      String proxyPortStr = System.getProperty(PROXY_PORT, "0");
+      int proxyPort = 0;
+      try {
+        proxyPort = Integer.parseInt(proxyPortStr);
+      } catch (NumberFormatException e) {
+        throw new IllegalArgumentException(
+            "Invalid proxy port: '" + proxyPortStr + "'. Proxy port must be a valid integer.", e);
+      }
+      String nonProxyHosts = System.getProperty(NON_PROXY_HOSTS, "");
+      String proxyUser = System.getProperty(HTTP_PROXY_USER, "");
+      String proxyPassword = System.getProperty(HTTP_PROXY_PASSWORD, "");
+
+      LOGGER.trace(
+          "Creating HTTP client settings key with proxy configuration from system properties for"
+              + " account: {}. Proxy host: {}, port: {}, user: {}, nonProxyHosts: {}",
+          accountName,
+          proxyHost,
+          proxyPort,
+          isNullOrEmpty(proxyUser) ? "not set" : "set",
+          isNullOrEmpty(nonProxyHosts) ? "not set" : nonProxyHosts);
+      return new HttpClientSettingsKey(
+          accountName, proxyHost, proxyPort, nonProxyHosts, proxyUser, proxyPassword);
+    }
 
-    return httpClient;
+    // No proxy configuration
+    LOGGER.debug(
+        "Creating HTTP client settings key without proxy configuration for account: {}",
+        accountName);
+    return new HttpClientSettingsKey(accountName);
   }
 
-  private static final Logger LOGGER = LoggerFactory.getLogger(HttpUtil.class);
-
-  private static void initHttpClient(String accountName) {
+  /** Build HttpClient based on the settings key */
+  private static CloseableHttpClient buildHttpClient(HttpClientSettingsKey key) {
+    LOGGER.info("Building new HTTP client");
 
     Security.setProperty("ocsp.enable", "true");
 
@@ -167,34 +278,49 @@ private static void initHttpClient(String accountName) {
             .setDefaultRequestConfig(requestConfig);
 
     // proxy settings
-    if ("true".equalsIgnoreCase(System.getProperty(USE_PROXY)) && !shouldBypassProxy(accountName)) {
-      if (System.getProperty(PROXY_PORT) == null) {
+    if (key.usesProxy()) {
+      LOGGER.debug(
+          "Configuring HTTP client with proxy settings. Host: {}, Port: {}",
+          key.getProxyHost(),
+          key.getProxyPort());
+
+      if (isNullOrEmpty(key.getProxyHost())) {
         throw new IllegalArgumentException(
-            "proxy port number is not provided, please assign proxy port to http.proxyPort option");
+            "proxy host IP is not provided, please assign proxy host IP to http.proxyHost option");
       }
-      if (System.getProperty(PROXY_HOST) == null) {
+      if (key.getProxyPort() <= 0) {
         throw new IllegalArgumentException(
-            "proxy host IP is not provided, please assign proxy host IP to http.proxyHost option");
+            "proxy port number is not provided, please assign proxy port to http.proxyPort option");
       }
-      String proxyHost = System.getProperty(PROXY_HOST);
-      int proxyPort = Integer.parseInt(System.getProperty(PROXY_PORT));
+
+      String proxyHost = key.getProxyHost();
+      int proxyPort = key.getProxyPort();
       HttpHost proxy = new HttpHost(proxyHost, proxyPort, PROXY_SCHEME);
       DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
       clientBuilder = clientBuilder.setRoutePlanner(routePlanner);
 
       // Check if proxy username and password are set
-      final String proxyUser = System.getProperty(HTTP_PROXY_USER);
-      final String proxyPassword = System.getProperty(HTTP_PROXY_PASSWORD);
+      final String proxyUser = key.getProxyUser();
+      final String proxyPassword = key.getProxyPassword();
       if (!isNullOrEmpty(proxyUser) && !isNullOrEmpty(proxyPassword)) {
+        LOGGER.debug("Configuring HTTP client with proxy authentication credentials");
         Credentials credentials = new UsernamePasswordCredentials(proxyUser, proxyPassword);
         AuthScope authScope = new AuthScope(proxyHost, proxyPort);
         CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
         credentialsProvider.setCredentials(authScope, credentials);
         clientBuilder = clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+      } else {
+        LOGGER.debug(
+            "HTTP client configured with proxy but no authentication credentials provided");
       }
+    } else {
+      LOGGER.debug("Configuring HTTP client without proxy settings");
     }
 
-    httpClient = clientBuilder.build();
+    CloseableHttpClient httpClient = clientBuilder.build();
+    initIdleConnectionMonitoringThread();
+    LOGGER.trace("Successfully built new HTTP client for key: {}", key);
+    return httpClient;
   }
 
   /** Starts a daemon thread to monitor idle connections in http connection manager. */
@@ -324,6 +450,8 @@ private static String getRequestUriFromContext(final HttpContext httpContext) {
   public static Properties generateProxyPropertiesForJDBC() {
     Properties proxyProperties = new Properties();
     if (Boolean.parseBoolean(System.getProperty(USE_PROXY))) {
+      LOGGER.debug("Generating proxy properties for JDBC from system properties");
+
       if (isNullOrEmpty(System.getProperty(PROXY_PORT))) {
         throw new IllegalArgumentException(
             "proxy port number is not provided, please assign proxy port to http.proxyPort option");
@@ -340,19 +468,31 @@ public static Properties generateProxyPropertiesForJDBC() {
       proxyProperties.put(
           SFSessionProperty.PROXY_PORT.getPropertyKey(), System.getProperty(PROXY_PORT));
 
+      LOGGER.debug(
+          "Generated proxy properties - Host: {}, Port: {}",
+          System.getProperty(PROXY_HOST),
+          System.getProperty(PROXY_PORT));
+
       // Check if proxy username and password are set
       final String proxyUser = System.getProperty(HTTP_PROXY_USER);
       final String proxyPassword = System.getProperty(HTTP_PROXY_PASSWORD);
       if (!isNullOrEmpty(proxyUser) && !isNullOrEmpty(proxyPassword)) {
         proxyProperties.put(SFSessionProperty.PROXY_USER.getPropertyKey(), proxyUser);
         proxyProperties.put(SFSessionProperty.PROXY_PASSWORD.getPropertyKey(), proxyPassword);
+        LOGGER.debug("Added proxy authentication credentials to generated properties");
+      } else {
+        LOGGER.debug("No proxy authentication credentials found in system properties");
       }
 
       // Check if http.nonProxyHosts was set
       final String nonProxyHosts = System.getProperty(NON_PROXY_HOSTS);
       if (!isNullOrEmpty(nonProxyHosts)) {
         proxyProperties.put(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey(), nonProxyHosts);
+        LOGGER.debug("Added nonProxyHosts to generated properties: {}", nonProxyHosts);
       }
+    } else {
+      LOGGER.debug(
+          "System property {} is not set to true, generating empty proxy properties", USE_PROXY);
     }
     return proxyProperties;
   }
@@ -420,7 +560,33 @@ private boolean isShutdown() {
 
   /** Shuts down the daemon thread. */
   public static void shutdownHttpConnectionManagerDaemonThread() {
-    idleConnectionMonitorThread.shutdown();
+    idleConnectionMonitorThreadLock.lock();
+    try {
+      if (idleConnectionMonitorThread != null) {
+        idleConnectionMonitorThread.shutdown();
+        idleConnectionMonitorThread = null;
+      }
+    } finally {
+      idleConnectionMonitorThreadLock.unlock();
+    }
+  }
+
+  /** Close all cached HTTP clients and clear the cache */
+  public static void closeAllHttpClients() {
+    // for use in test
+    LOGGER.info(
+        "Closing all cached HTTP clients. Total clients in cache: {}", httpClientCache.size());
+    for (Map.Entry<HttpClientSettingsKey, CloseableHttpClient> entry : httpClientCache.entrySet()) {
+      try {
+        LOGGER.debug("Closing HTTP client for key: {}", entry.getKey());
+        entry.getValue().close();
+      } catch (Exception e) {
+        LOGGER.warn("Error closing HTTP client for key: {}", entry.getKey(), e);
+      }
+    }
+    httpClientCache.clear();
+    LOGGER.info("All cached HTTP clients closed and cache cleared");
+    shutdownHttpConnectionManagerDaemonThread();
   }
 
   /** Create Pool stats for a route */
@@ -441,25 +607,60 @@ private static String createPoolStatsInfo(String title, PoolStats poolStats) {
 
   /**
    * Changes the account name to the format accountName.snowflakecomputing.com then returns a
-   * boolean to indicate if we should go through a proxy or not.
+   * boolean to indicate if we should go through a proxy or not. This is a convenience method for
+   * existing tests that calls the two-parameter version with null properties.
    */
   public static Boolean shouldBypassProxy(String accountName) {
+    return shouldBypassProxy(accountName, null);
+  }
+
+  /**
+   * Changes the account name to the format accountName.snowflakecomputing.com then returns a
+   * boolean to indicate if we should go through a proxy or not. Checks both system properties and
+   * provided proxy properties for non-proxy hosts configuration.
+   */
+  public static Boolean shouldBypassProxy(String accountName, Properties proxyProperties) {
     String targetHost = accountName + SNOWFLAKE_DOMAIN_NAME;
-    return System.getProperty(NON_PROXY_HOSTS) != null && isInNonProxyHosts(targetHost);
+
+    // Check system properties first
+    String systemNonProxyHosts = System.getProperty(NON_PROXY_HOSTS);
+    if (systemNonProxyHosts != null && isInNonProxyHosts(targetHost, systemNonProxyHosts)) {
+      LOGGER.info(
+          "Account {} ({}) matches system nonProxyHosts pattern. Bypassing proxy.",
+          accountName,
+          targetHost);
+      return true;
+    }
+
+    // Check SFSessionProperty.NON_PROXY_HOSTS from proxyProperties if available
+    if (proxyProperties != null) {
+      String sessionNonProxyHosts =
+          proxyProperties.getProperty(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey());
+      if (sessionNonProxyHosts != null && isInNonProxyHosts(targetHost, sessionNonProxyHosts)) {
+        LOGGER.info(
+            "Account {} ({}) matches session nonProxyHosts pattern. Bypassing proxy.",
+            accountName,
+            targetHost);
+        return true;
+      }
+    }
+
+    LOGGER.debug(
+        "Account {} ({}) does not match any nonProxyHosts pattern.", accountName, targetHost);
+    return false;
   }
 
   /**
    * The target hostname input is compared with the hosts in the '|' separated list provided by the
-   * http.nonProxyHosts parameter using regex. The nonProxyHosts will be used as our Patterns, so we
-   * need to replace the '.' and '*' characters since those are special regex constructs that mean
-   * 'any character,' and 'repeat 0 or more times.'
+   * nonProxyHosts parameter using regex. The nonProxyHosts will be used as our Patterns, so we need
+   * to replace the '.' and '*' characters since those are special regex constructs that mean 'any
+   * character,' and 'repeat 0 or more times.'
    */
-  private static Boolean isInNonProxyHosts(String targetHost) {
-    String nonProxyHosts =
-        System.getProperty(NON_PROXY_HOSTS).replace(".", "\\.").replace("*", ".*");
-    String[] nonProxyHostsArray = nonProxyHosts.split("\\|");
-    for (String i : nonProxyHostsArray) {
-      if (Pattern.compile(i).matcher(targetHost).matches()) {
+  private static Boolean isInNonProxyHosts(String targetHost, String nonProxyHosts) {
+    String escapedNonProxyHosts = nonProxyHosts.replace(".", "\\.").replace("*", ".*");
+    String[] nonProxyHostsArray = escapedNonProxyHosts.split("\\|");
+    for (String pattern : nonProxyHostsArray) {
+      if (Pattern.compile(pattern).matcher(targetHost).matches()) {
         return true;
       }
     }
diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java
index a2653ab14..45f67a83d 100644
--- a/src/main/java/net/snowflake/ingest/utils/Utils.java
+++ b/src/main/java/net/snowflake/ingest/utils/Utils.java
@@ -139,7 +139,12 @@ public static Properties createProperties(Properties inputProp) {
           privateKeyPassphrase = val;
           break;
         default:
-          properties.put(key.toLowerCase(), val);
+          // Preserve case for proxy-related properties that HttpUtil expects
+          if (isProxyRelatedProperty(key)) {
+            properties.put(key, val);
+          } else {
+            properties.put(key.toLowerCase(), val);
+          }
       }
     }
 
@@ -215,6 +220,17 @@ public static Properties createProperties(Properties inputProp) {
     return properties;
   }
 
+  /** Check if a property key is proxy-related and should preserve its case */
+  private static boolean isProxyRelatedProperty(String key) {
+    return key.equals(SFSessionProperty.USE_PROXY.getPropertyKey())
+        || key.equals(SFSessionProperty.PROXY_HOST.getPropertyKey())
+        || key.equals(SFSessionProperty.PROXY_PORT.getPropertyKey())
+        || key.equals(SFSessionProperty.PROXY_USER.getPropertyKey())
+        || key.equals(SFSessionProperty.PROXY_PASSWORD.getPropertyKey())
+        || key.equals(SFSessionProperty.NON_PROXY_HOSTS.getPropertyKey())
+        || key.equals(SFSessionProperty.PROXY_PROTOCOL.getPropertyKey());
+  }
+
   /** Construct account url from input schema, host and port */
   public static String constructAccountUrl(String scheme, String host, int port) {
     return String.format("%s://%s:%d", scheme, host, port);
diff --git a/src/test/java/net/snowflake/ingest/utils/HttpUtilCacheTest.java b/src/test/java/net/snowflake/ingest/utils/HttpUtilCacheTest.java
new file mode 100644
index 000000000..61d78b915
--- /dev/null
+++ b/src/test/java/net/snowflake/ingest/utils/HttpUtilCacheTest.java
@@ -0,0 +1,108 @@
+package net.snowflake.ingest.utils;
+
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+
+import java.util.Properties;
+import net.snowflake.client.core.SFSessionProperty;
+import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
+import org.junit.After;
+import org.junit.Test;
+
+/**
+ * Test class for HttpUtil caching functionality. Tests verify that HTTP clients are properly cached
+ * based on account name and proxy settings. Note: Both JDBC and streaming clients now use unified
+ * SFSessionProperty constants for proxy configuration.
+ */
+public class HttpUtilCacheTest {
+
+  @After
+  public void tearDown() {
+    // Clean up after each test
+    HttpUtil.closeAllHttpClients();
+  }
+
+  @Test
+  public void testHttpClientCachingWithSameAccount() {
+    String accountName = "testaccount";
+
+    // Get HTTP client twice with same account
+    CloseableHttpClient client1 = HttpUtil.getHttpClient(accountName);
+    CloseableHttpClient client2 = HttpUtil.getHttpClient(accountName);
+
+    // Should return the same instance
+    assertSame("HTTP clients should be the same for same account", client1, client2);
+  }
+
+  @Test
+  public void testHttpClientCachingWithDifferentAccounts() {
+    String accountName1 = "testaccount1";
+    String accountName2 = "testaccount2";
+
+    // Get HTTP clients for different accounts
+    CloseableHttpClient client1 = HttpUtil.getHttpClient(accountName1);
+    CloseableHttpClient client2 = HttpUtil.getHttpClient(accountName2);
+
+    // Should return different instances
+    assertNotSame("HTTP clients should be different for different accounts", client1, client2);
+  }
+
+  @Test
+  public void testHttpClientCachingWithSameProxySettings() {
+    String accountName = "testaccount";
+    Properties proxyProps = new Properties();
+    proxyProps.setProperty(SFSessionProperty.USE_PROXY.getPropertyKey(), "true");
+    proxyProps.setProperty(SFSessionProperty.PROXY_HOST.getPropertyKey(), "proxy.example.com");
+    proxyProps.setProperty(SFSessionProperty.PROXY_PORT.getPropertyKey(), "8080");
+
+    // Get HTTP client twice with same proxy settings
+    CloseableHttpClient client1 = HttpUtil.getHttpClient(accountName, proxyProps);
+    CloseableHttpClient client2 = HttpUtil.getHttpClient(accountName, proxyProps);
+
+    // Should return the same instance
+    assertSame("HTTP clients should be the same for same proxy settings", client1, client2);
+  }
+
+  @Test
+  public void testHttpClientCachingWithDifferentProxySettings() {
+    String accountName = "testaccount";
+
+    Properties proxyProps1 = new Properties();
+    proxyProps1.setProperty(SFSessionProperty.USE_PROXY.getPropertyKey(), "true");
+    proxyProps1.setProperty(SFSessionProperty.PROXY_HOST.getPropertyKey(), "proxy1.example.com");
+    proxyProps1.setProperty(SFSessionProperty.PROXY_PORT.getPropertyKey(), "8080");
+
+    Properties proxyProps2 = new Properties();
+    proxyProps2.setProperty(SFSessionProperty.USE_PROXY.getPropertyKey(), "true");
+    proxyProps2.setProperty(SFSessionProperty.PROXY_HOST.getPropertyKey(), "proxy2.example.com");
+    proxyProps2.setProperty(SFSessionProperty.PROXY_PORT.getPropertyKey(), "8080");
+
+    // Get HTTP clients with different proxy settings
+    CloseableHttpClient client1 = HttpUtil.getHttpClient(accountName, proxyProps1);
+    CloseableHttpClient client2 = HttpUtil.getHttpClient(accountName, proxyProps2);
+
+    // Should return different instances
+    assertNotSame(
+        "HTTP clients should be different for different proxy settings", client1, client2);
+  }
+
+  @Test
+  public void testHttpClientCachingWithProxyAndNoProxy() {
+    String accountName = "testaccount";
+
+    Properties proxyProps = new Properties();
+    proxyProps.setProperty(SFSessionProperty.USE_PROXY.getPropertyKey(), "true");
+    proxyProps.setProperty(SFSessionProperty.PROXY_HOST.getPropertyKey(), "proxy.example.com");
+    proxyProps.setProperty(SFSessionProperty.PROXY_PORT.getPropertyKey(), "8080");
+
+    // Get HTTP client with proxy and without proxy
+    CloseableHttpClient clientWithProxy = HttpUtil.getHttpClient(accountName, proxyProps);
+    CloseableHttpClient clientWithoutProxy = HttpUtil.getHttpClient(accountName);
+
+    // Should return different instances
+    assertNotSame(
+        "HTTP clients should be different for proxy vs no proxy",
+        clientWithProxy,
+        clientWithoutProxy);
+  }
+}

From d255ebc74a88946e39d66e8c80d70063ff9ca0d4 Mon Sep 17 00:00:00 2001
From: Sangeet Mishra <mail2sangeetmishra@gmail.com>
Date: Thu, 19 Jun 2025 19:26:27 +0530
Subject: [PATCH 354/356] changes

---
 .semaphore/semaphore.yml | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
index 4968487b6..9bfa7e7f4 100644
--- a/.semaphore/semaphore.yml
+++ b/.semaphore/semaphore.yml
@@ -58,7 +58,25 @@ blocks:
           commands:
             - ci-tools ci-update-version
             - ci-tools ci-push-tag
-            - mvn -U -Dmaven.wagon.http.retryHandler.count=10 --batch-mode -DaltDeploymentRepository=confluent-codeartifact-internal::default::https://confluent-519856050701.d.codeartifact.us-west-2.amazonaws.com/maven/maven-snapshots/
+            - mvn -U -Dmaven.wagon.http.retryHandler.count=10 --batch-mode -DaltDeploymentRepository=confluent-codeartifact-internal::default::https://confluent-519856050701.d.codeartifact.us-west-2.amazonaws.com/maven/maven-releases/ -DrepositoryId=confluent-codeartifact-internal deploy -DskipTests
+
+  - name: Snapshot Release
+    dependencies: [ "Test" ]
+    run:
+      when: "branch != 'master' and branch !~ '[0-9]+\\.[0-9]+\\.x'"
+    task:
+      jobs:
+        - name: Snapshot Release
+          commands:
+            - export COMMIT_SHA=$(git rev-parse --short HEAD)
+            - echo "Commit SHA = ${COMMIT_SHA}"
+            - export CURRENT_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+            - echo "Current version = ${CURRENT_VERSION}"
+            - mvn versions:set -DnewVersion="${CURRENT_VERSION}-${COMMIT_SHA}-SNAPSHOT" -DgenerateBackupPoms=false
+            - echo "New version = ${CURRENT_VERSION}-${COMMIT_SHA}-SNAPSHOT"
+            - >
+              mvn -U -Dmaven.wagon.http.retryHandler.count=10 --batch-mode
+              -DaltDeploymentRepository=confluent-codeartifact-internal::default::https://confluent-519856050701.d.codeartifact.us-west-2.amazonaws.com/maven/maven-snapshots/
               -DrepositoryId=confluent-codeartifact-internal deploy -DskipTests
 
 after_pipeline:

From 294fe3f9f429d086da3a223e5af9e4bb50c7450b Mon Sep 17 00:00:00 2001
From: Sangeet Mishra <mail2sangeetmishra@gmail.com>
Date: Thu, 19 Jun 2025 22:10:15 +0530
Subject: [PATCH 355/356] add to package path

---
 service.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/service.yml b/service.yml
index ec8002f66..ba20b830d 100644
--- a/service.yml
+++ b/service.yml
@@ -20,3 +20,4 @@ code_artifact:
   enable: true
   package_paths:
     - maven-snapshots/maven/io.confluent/snowflake-ingest-sdk
+    - maven-releases/maven/io.confluent/snowflake-ingest-sdk

From 3219ba830058ab803347f37d8a2ab6bdbb711499 Mon Sep 17 00:00:00 2001
From: Sangeet Mishra <mail2sangeetmishra@gmail.com>
Date: Fri, 20 Jun 2025 08:08:07 +0530
Subject: [PATCH 356/356] add some release related stuff

---
 .semaphore/semaphore.yml                                 | 9 +++------
 e2e-jar-test/pom.xml                                     | 2 +-
 pom.xml                                                  | 2 +-
 service.yml                                              | 1 +
 .../net/snowflake/ingest/connection/RequestBuilder.java  | 2 +-
 5 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
index 9bfa7e7f4..62876027b 100644
--- a/.semaphore/semaphore.yml
+++ b/.semaphore/semaphore.yml
@@ -63,7 +63,7 @@ blocks:
   - name: Snapshot Release
     dependencies: [ "Test" ]
     run:
-      when: "branch != 'master' and branch !~ '[0-9]+\\.[0-9]+\\.x'"
+      when: "branch =~ '.*-test-release$'"
     task:
       jobs:
         - name: Snapshot Release
@@ -74,10 +74,7 @@ blocks:
             - echo "Current version = ${CURRENT_VERSION}"
             - mvn versions:set -DnewVersion="${CURRENT_VERSION}-${COMMIT_SHA}-SNAPSHOT" -DgenerateBackupPoms=false
             - echo "New version = ${CURRENT_VERSION}-${COMMIT_SHA}-SNAPSHOT"
-            - >
-              mvn -U -Dmaven.wagon.http.retryHandler.count=10 --batch-mode
-              -DaltDeploymentRepository=confluent-codeartifact-internal::default::https://confluent-519856050701.d.codeartifact.us-west-2.amazonaws.com/maven/maven-snapshots/
-              -DrepositoryId=confluent-codeartifact-internal deploy -DskipTests
+            - mvn -U -Dmaven.wagon.http.retryHandler.count=10 --batch-mode -DaltDeploymentRepository=confluent-codeartifact-internal::default::https://confluent-519856050701.d.codeartifact.us-west-2.amazonaws.com/maven/maven-snapshots/ -DrepositoryId=confluent-codeartifact-internal deploy -DskipTests
 
 after_pipeline:
   task:
@@ -96,4 +93,4 @@ after_pipeline:
           - checkout
           - sem-version java 11
           - artifact pull workflow target
-          - emit-sonarqube-data --run_only_sonar_scan
\ No newline at end of file
+          - emit-sonarqube-data --run_only_sonar_scan
diff --git a/e2e-jar-test/pom.xml b/e2e-jar-test/pom.xml
index 9a6933654..e9fb43592 100644
--- a/e2e-jar-test/pom.xml
+++ b/e2e-jar-test/pom.xml
@@ -27,7 +27,7 @@
             <dependency>
                 <groupId>net.snowflake</groupId>
                 <artifactId>snowflake-ingest-sdk</artifactId>
-                <version>2.1.5</version>
+                <version>2.0.0</version>
             </dependency>
 
             <dependency>
diff --git a/pom.xml b/pom.xml
index ec3f345d4..58cd1f485 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
   <!-- Arifact name and version information -->
   <groupId>io.confluent</groupId>
   <artifactId>snowflake-ingest-sdk</artifactId>
-  <version>2.1.5</version>
+  <version>2.0.0</version>
   <packaging>jar</packaging>
   <name>Snowflake Ingest SDK</name>
   <description>Snowflake Ingest SDK</description>
diff --git a/service.yml b/service.yml
index ba20b830d..e217c3de9 100644
--- a/service.yml
+++ b/service.yml
@@ -12,6 +12,7 @@ semaphore:
     - /^\d+\.\d+\.x$/
     - /v\d+\.\d+\.\d+\-hotfix\-x/
     - /^gh-readonly-queue.*/
+    - /.*-test-release$/
 git:
   enable: true
 github:
diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
index affee8161..3fdb8a660 100644
--- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
+++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java
@@ -110,7 +110,7 @@ public class RequestBuilder {
   // Don't change!
   public static final String CLIENT_NAME = "SnowpipeJavaSDK";
 
-  public static final String DEFAULT_VERSION = "2.1.5";
+  public static final String DEFAULT_VERSION = "2.0.0";
 
   public static final String JAVA_USER_AGENT = "JAVA";