feat: Add feature for copying backups (#1153)

* src test and protos copied over

* lint fix

* implemented copy backup

* part of test done

* workaround

* corrections to get code working

* Add expire time

* This allows the copy backup endpoint to work

* Working system test

* delete try block

* tests pass with new refactor

* Create backup and copy on another cluster

* modifications to second test

* copy backup unit test

* Revert "lint fix"

This reverts commit 576550d.

* Revert "src test and protos copied over"

This reverts commit 46ce5b8.

* First correction

* Check the list of backups

* fetched backup

* get name and id

* Remove only

* refactor request mock

* Add unit test for copying backup to another projec

* feat: add experimental reverse scan for public preview

PiperOrigin-RevId: 543539118

Source-Link: googleapis/googleapis@ae18706

Source-Link: googleapis/googleapis-gen@5d05516
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiNWQwNTUxNmY4NGU1M2FhYmE2M2E0Yjg3NjdmZjk1NWFjNWJiNGE4NyJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* feat: Increase the maximum retention period for a Cloud Bigtable backup from 30 days to 90 days

PiperOrigin-RevId: 544356969

Source-Link: googleapis/googleapis@c35889a

Source-Link: googleapis/googleapis-gen@c00326e
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiYzAwMzI2ZWM3ODU2NWI1ZDE2ZjkyYzg0NWZmMGJiMThmMTFjYTA1ZCJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* docs: fix formatting for reversed order field example

PiperOrigin-RevId: 547553954

Source-Link: googleapis/googleapis@c4e6427

Source-Link: googleapis/googleapis-gen@f552269
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiZjU1MjI2OTYwOWQ0MTgzNTQ2NTQzYmZlM2EwMjJmNTQ0ZDRmNWJkYiJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* feat: add last_scanned_row_key feature

PiperOrigin-RevId: 551191182

Source-Link: googleapis/googleapis@51e04ba

Source-Link: googleapis/googleapis-gen@4b90e8e
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiNGI5MGU4ZWFkNDQ3N2VmZjk2YzMxYjliMGZkZWYzNmVkOTc1YjE1ZiJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* fix: fix typings for IAM methods
docs: fixed links in the generated Markdown documentation

PiperOrigin-RevId: 551610576

Source-Link: googleapis/googleapis@73b1313

Source-Link: googleapis/googleapis-gen@8bec066
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiOGJlYzA2NjQ5MmE2ZGEyODU1YjFiOGNlNTYyNjY0YzBhNmIzMGIwMSJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* test: disable retry-request for streaming tests

PiperOrigin-RevId: 554648220

Source-Link: googleapis/googleapis@53cd9ad

Source-Link: googleapis/googleapis-gen@7e8867e
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiN2U4ODY3ZWZiZWQ3ZGJmZTVlZjZlYzNjMmM5MmE0YmNlNDI4MGY3YSJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* feat: publish CopyBackup protos to external customers

PiperOrigin-RevId: 557192020

Source-Link: googleapis/googleapis@b4c238f

Source-Link: googleapis/googleapis-gen@feccb30
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiZmVjY2IzMGUzMTc3ZGE4YjdiN2U2ODE0OWNhNGJiOTE0ZjhmYWYyYSJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* fix: simplify logic for HTTP/1.1 REST fallback option

For the `fallback` parameter, all values considered as `true`
in Boolean context will enable HTTP/1.1 REST fallback,
since the other fallback transport, proto over HTTP, is
removed from `google-gax` v4.

PiperOrigin-RevId: 559812260

Source-Link: googleapis/googleapis@6a6fd29

Source-Link: googleapis/googleapis-gen@56c1665
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiNTZjMTY2NTdlN2E1OTEyMmIxZGE5NDc3MWE5ZWY0MDk4OWMyODJjMCJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* fix: add feature flag for improved mutate rows throttling

PiperOrigin-RevId: 565090488

Source-Link: googleapis/googleapis@e8a136f

Source-Link: googleapis/googleapis-gen@9a8dcca
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiOWE4ZGNjYTBmYjIxMTc2MjhhMWE2YTZjMzYyNWE2YWEzMmZjMmY3NSJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* run lint

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* build: update typescript generator version to publish in dual format (ESM)

PiperOrigin-RevId: 568643156

Source-Link: googleapis/googleapis@f95afc0

Source-Link: googleapis/googleapis-gen@bbd2c49
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiYmJkMmM0OWQyZTQyM2E4Y2U1Y2M4NTYyNzQwMmQ1MTJhZWVmYzU4YiJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* feat: Add support for Cloud Bigtable Request Priorities in App Profiles

PiperOrigin-RevId: 571158646

Source-Link: googleapis/googleapis@bc3c83b

Source-Link: googleapis/googleapis-gen@93366e8
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiOTMzNjZlODRlNGU2ODYxZTJlNTgwZWIwMDA3MjFkOTliZjU0YTBhNCJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* build: update Node.js generator to compile protos

PiperOrigin-RevId: 582493526

Source-Link: googleapis/googleapis@7c4e4b5

Source-Link: googleapis/googleapis-gen@368cfb6
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiMzY4Y2ZiNjUxMDE2ZDZhOTNjYTZlNDg4Y2JjMzRlMmQxZDlkMjEyYyJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* Use the destination backup expiry time

Expiry time of the destination should be used for the call. Not for the source expiry time.

* Use the config instead for copy backup

Use the config and not the backup object for copying backups. The expiry time is needed so a backup object can’t be used because the metadata cannot be introspected.

* Rename CopyBackupConfig

Rename it to DestinationBackupConfig.

* Fix signature of copyBackup

Fix the signature of copyBackup to include the fact that the config now has the gax options.

* Change the copy signature to require callback

If the callback is not required then all calls match first signature overload and async calls return void which isn’t correct.

* Some debugging on the system tests

The system tests were giving an error about the source expiry time and now this is fixed.

* DestinationBackupConfig

Don’t use the name DestinationBackupConfig anymore.

* lint fix

* Move the config and create a generic callback

Create a generic callback data structure in order to avoid callback and promise data structures not matching up. Also move the configs to a different file.

* Eliminate the generic callback data structure

Eliminate the generic data structure as it can’t be used to capture the pattern we need exactly. Also modify the way that errors are passed into the callback function to be more consistent.

* Get the first copy backup test working

Copy the backup into the same cluster test is working. Still need to work on copy backup for test that looks at a different cluster to get that test working.

* Fix different cluster, different instance test

Fix the issue preventing the second test from working by allowing the test function to specify the instance where the new backups should be created.

* Add another test for copy backup

The test should address a backup copied to a different cluster, but the same instance.

* Add a test case for copying to another project

Test case for copying to another project uses environment variable to point to new project for CI pipeline.

* Restructure tests for multiple expire time inputs

Take one of the tests and structure it into a describe block instead. Each test in this block will now run the old test with a different expiry time format.

* Various changes to test interaction with copy back

Rename backupId to id. id is a simpler name and it is the name that is used everywhere. Rename a function to testWithExpiryTimes. Modify source code for copy function to return a backup with the new id. Add test stub for restore backup.

* Finish the restore tests and modify copy backup

The restore test should test to see if a table can be restored. It should also return a backup object that matches the new backup created.

* Add comment describing second argument

Second argument is a backup that corresponds to a new backup. This needs to be explained in a comment.

* Remove TODO

The TODO is done

* Modify copy backup unit test

Modify the copy backup unit test to compare against copying the a destination with customized cluster, backup, instance and project.

* Modify the name of the test

The test is not necessarily about copying to a different project. It is more a test about copying to a specific project.

* Test for gax options in the test

Make sure that the gax options get passed down to the request layer.

* TODO is done

The promise and callbacks follow the same pattern that is applied everywhere.

* Change comment slightly

It is actually more of a check than an action that ensures something.

* Eliminate TODO

The id is required. It can’t be generated automatically by the server. It needs to be provided.

* Fix documentation for source function

Add description for the parameters and explain what the function does.

* Move copy backup code to proper place

The tests were originally placed in a block at the very beginning so that they were easier to work through. This change moves them to the proper place.

* Use cluster instead of parent

cluster should be used instead of parent to specify the destination cluster.

* Change parent to cluster

cluster name should be used instead of parent to match the design doc.

* A couple cleanup changes

Id is required so no question mark needed. Also inline the function that replaces the project name.

* Use parent instead of cluster

This test was accidentally changed by a refactor and should not change further.

* Eliminate unused time references

Variables were created so that time can be used in a block of tests. That block of tests were moved so these. variables are no longer needed.

* This test should not have changed

Use parent instead of cluster because that is what it was before.

* Eliminate unused import

CreateBackupConfig is not used anywhere.

* Remove assert statement that is not needed

* Move two lines of code to location used

General cleanup for readability

* Improve readability

Reduce number of lines required in testWithExpiryTimes function. This makes the function easier to read.

* Inline variables instead of using them explicitly

Don’t use the variables explicitly. Inline them instead.

* Generate the backup id inline

An extra variable is not needed for this.

* readability - indent code so it is separate

Also inline the id generation as it is not used elsewhere.

* Indent block to separate check

Separate expiry time check from the rest of the code.

* indent cluster creation, expiry time check

Indenting the code makes it easier to see variable relationships

* Another test cleanup

Inline Bigtable instance creation options. Indent code for checking expiry time.

* Add comment

Comment is for test that copies a table onto another project.

* Indent code for table data insertion

Indenting this code shows it is not referenced later.

* Rename variables to distinguish operations

Differentiate copy operation from create operation.

* Eliminate unused imports

* should be lower case

* Change comment to respect alignment

* Inline callback, config and cluster

Inline variables for better readability

* Remove unused operation

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* Delete an instance

Delete the instance after it is used just like the other test.

* Inline generate id

A generateId function I need to inline.

* Add second project to config

Import the second project to use in the integration tests

* Rename the function to setBackupExpiryTime

Rename this function to be more specific.

* Replace the project for backup path

Currently this test is too specific. It does not work if a specific project is specified in the client so we must only check the rest of the backup path.

* Run the linter

Linter removes unnecessary comma.

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* Update comments for the copy function

This function needs to be more clear and explain exactly what it does without ambiguity.

* Rename function that sets correct value of expiry

expiryTime should be in the right format so rename function to more clearly indicate what it is doing.

* Add an environment variable for second project

The variable for the second project is needed for the the integration tests to run properly for the copy backup test that copies to another project.

* Add log for second project

See what kokoro prints out.

* Remove the console log

This was just for testing. Remove this now.

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* Rename bigtable to bigtableSecondaryProject

* PR follow-ups - inline code and rename variables

* Remove unused import

* Add comments for expire time

* Add better garbage cleanups of the backups

* These comments are not needed anymore.

* Remove only

* Update system-test/bigtable.ts

Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com>

* Update system-test/bigtable.ts

Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com>

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
Co-authored-by: sofisl <55454395+sofisl@users.noreply.github.com>
Co-authored-by: Sofia Leon <sofialeon@google.com>
Co-authored-by: meredithslota <meredithslota@google.com>
Co-authored-by: Leah E. Cole <6719667+leahecole@users.noreply.github.com>

src/backup.ts

@@ -37,6 +37,15 @@ import {
 } from './cluster';
 import {CallOptions, LROperation, Operation, ServiceError} from 'google-gax';
 import {Instance} from './instance';
+import {ClusterUtils} from './utils/cluster';
+
+export type CopyBackupResponse = GenericBackupPromise<Operation>;
+export type CopyBackupCallback = GenericBackupCallback<Operation>;
+export interface CopyBackupConfig extends ModifiableBackupFields {
+  cluster: Cluster;
+  gaxOptions?: CallOptions;
+  id: string;
+}
 
 type IEmpty = google.protobuf.IEmpty;
 export type IBackup = google.bigtable.admin.v2.IBackup;
@@ -59,6 +68,7 @@ export interface GenericBackupCallback<T> {
     apiResponse?: T | null
   ): void;
 }
+export type GenericBackupPromise<T> = [Backup, T];
 
 export type DeleteBackupCallback = (
   err: ServiceError | null,
@@ -243,6 +253,50 @@ Please use the format 'my-backup' or '${cluster.name}/backups/my-backup'.`);
     });
   }
 
+  /**
+   * When this backup object represents a backup that has already been created,
+   * copy will copy this created backup to the location and with the settings
+   * specified by the config parameter. After running this function the original
+   * backup will exist as well as a second backup matching the parameters given
+   * by the config argument.
+   *
+   * @param {CopyBackupConfig} [config] The config that specifies all of the
+   * information about the destination backup which is the new backup that gets
+   * created as a result of calling copy.
+   * @param {CopyBackupCallback} [callback] The callback function that passes an
+   * error or results back to the user.
+   */
+  copy(config: CopyBackupConfig, callback: CopyBackupCallback): void;
+  copy(config: CopyBackupConfig): Promise<CopyBackupResponse>;
+  copy(
+    config: CopyBackupConfig,
+    callback?: CopyBackupCallback
+  ): void | Promise<CopyBackupResponse> {
+    const reqOpts = {
+      parent: config.cluster.name,
+      backupId: config.id,
+      sourceBackup: `${this.cluster.name}/backups/${this.id}`,
+      expireTime: config?.expireTime,
+    };
+    ClusterUtils.formatBackupExpiryTime(reqOpts);
+    this.bigtable.request(
+      {
+        client: 'BigtableTableAdminClient',
+        method: 'copyBackup',
+        reqOpts,
+        gaxOpts: config.gaxOptions,
+      },
+      (err, ...args) => {
+        if (err) {
+          callback!(err, undefined, ...args);
+          return;
+        }
+        // Second argument is a backup for the new backup id
+        callback!(null, config.cluster.backup(config.id), ...args);
+      }
+    );
+  }
+
   create(config: CreateBackupConfig, callback?: CreateBackupCallback): void;
   create(config: CreateBackupConfig): Promise<CreateBackupResponse>;
   /**

src/cluster.ts

@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import {PreciseDate} from '@google-cloud/precise-date';
 import {promisifyAll} from '@google-cloud/promisify';
 import {CallOptions, LROperation, Operation, ServiceError} from 'google-gax';
 
@@ -314,11 +313,7 @@ Please use the format 'my-cluster' or '${instance.name}/clusters/my-cluster'.`);
       },
     };
 
-    if (reqOpts.backup.expireTime instanceof Date) {
-      reqOpts.backup.expireTime = new PreciseDate(
-        reqOpts.backup.expireTime
-      ).toStruct();
-    }
+    ClusterUtils.formatBackupExpiryTime(reqOpts.backup);
 
     delete reqOpts.backup.table;
     delete reqOpts.backup.gaxOptions;

src/utils/cluster.ts

@@ -20,6 +20,8 @@ import {
   SetClusterMetadataOptions,
 } from '../cluster';
 import {google} from '../../protos/protos';
+import {ModifiableBackupFields} from '../backup';
+import {PreciseDate} from '@google-cloud/precise-date';
 
 export class ClusterUtils {
   static noConfigError =
@@ -163,4 +165,10 @@ export class ClusterUtils {
       updateMask: {paths: this.getUpdateMask(metadata)},
     };
   }
+
+  static formatBackupExpiryTime(backup: ModifiableBackupFields) {
+    if (backup.expireTime instanceof Date) {
+      backup.expireTime = new PreciseDate(backup.expireTime).toStruct();
+    }
+  }
 }