[NO ISSUE][STO] Remove Storage Auto Migration Code
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- It was decided to not support auto storage migration
from old to newer version at this point in time. This
change removes the storage migration code and its test.
Change-Id: I6faa816ca71913f18ff6017e86c8a29253728572
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2305
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MigrateStorageResourcesTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MigrateStorageResourcesTask.java
deleted file mode 100644
index bd64ea5..0000000
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MigrateStorageResourcesTask.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.app.nc.task;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.apache.asterix.common.api.INCLifecycleTask;
-import org.apache.asterix.common.api.INcApplicationContext;
-import org.apache.asterix.common.dataflow.DatasetLocalResource;
-import org.apache.asterix.common.storage.DatasetResourceReference;
-import org.apache.asterix.common.transactions.Checkpoint;
-import org.apache.asterix.common.transactions.ICheckpointManager;
-import org.apache.asterix.common.utils.StorageConstants;
-import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
-import org.apache.commons.io.FileUtils;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IIOManager;
-import org.apache.hyracks.api.io.IODeviceHandle;
-import org.apache.hyracks.api.service.IControllerService;
-import org.apache.hyracks.storage.common.ILocalResourceRepository;
-import org.apache.hyracks.storage.common.LocalResource;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-/**
- * Migrates a legacy storage structure to the current one
- */
-public class MigrateStorageResourcesTask implements INCLifecycleTask {
-
- private static final Logger LOGGER = LogManager.getLogger();
- private static final long serialVersionUID = 1L;
- public static final int LEGACY_RESOURCES_TREE_DEPTH_FROM_STORAGE_ROOT = 5;
-
- @Override
- public void perform(IControllerService cs) throws HyracksDataException {
- final INcApplicationContext appCtx = (INcApplicationContext) cs.getApplicationContext();
- final ICheckpointManager checkpointMgr = appCtx.getTransactionSubsystem().getCheckpointManager();
- final Checkpoint latestCheckpoint = checkpointMgr.getLatest();
- if (latestCheckpoint == null) {
- // nothing to migrate
- return;
- }
- final IIOManager ioManager = appCtx.getIoManager();
- final List<IODeviceHandle> ioDevices = ioManager.getIODevices();
- for (IODeviceHandle ioDeviceHandle : ioDevices) {
- final Path root = Paths.get(ioDeviceHandle.getMount().getAbsolutePath());
- if (!root.toFile().exists()) {
- continue;
- }
- try (Stream<Path> stream = Files.find(root, LEGACY_RESOURCES_TREE_DEPTH_FROM_STORAGE_ROOT,
- (path, attr) -> path.getFileName().toString().equals(StorageConstants.METADATA_FILE_NAME))) {
- final List<Path> resourceToMigrate = stream.map(Path::getParent).collect(Collectors.toList());
- for (Path src : resourceToMigrate) {
- final Path dest =
- migrateResourceMetadata(root.relativize(src), appCtx, latestCheckpoint.getStorageVersion());
- copyResourceFiles(root.resolve(src), root.resolve(dest),
- PersistentLocalResourceRepository.INDEX_COMPONENTS);
- FileUtils.deleteDirectory(root.resolve(src).toFile());
- }
- } catch (IOException e) {
- throw HyracksDataException.create(e);
- }
- }
- }
-
- /**
- * Migrates the resource metadata file at {@code resourcePath} to the new storage structure
- * and updates the migrated resource's metadata to reflect the new path.
- *
- * @param resourcePath
- * @param appCtx
- * @param resourceVersion
- * @return The migrated resource relative path
- * @throws HyracksDataException
- */
- private Path migrateResourceMetadata(Path resourcePath, INcApplicationContext appCtx, int resourceVersion)
- throws HyracksDataException {
- final ILocalResourceRepository localResourceRepository = appCtx.getLocalResourceRepository();
- final LocalResource srcResource = localResourceRepository.get(resourcePath.toFile().getPath());
- final DatasetLocalResource lsmResource = (DatasetLocalResource) srcResource.getResource();
- // recreate the resource with the new path and version
- final DatasetResourceReference lrr = DatasetResourceReference.of(srcResource, resourceVersion);
- final Path destPath = lrr.getRelativePath();
- final FileReference destDir = appCtx.getIoManager().resolve(destPath.toString());
- // ensure the new dest dir is empty
- if (destDir.getFile().exists()) {
- FileUtils.deleteQuietly(destDir.getFile());
- }
- lsmResource.setPath(destPath.toString());
-
- final LocalResource destResource =
- new LocalResource(srcResource.getId(), srcResource.getVersion(), srcResource.isDurable(), lsmResource);
- LOGGER.info(() -> "Migrating resource from: " + srcResource.getPath() + " to " + destResource.getPath());
- localResourceRepository.insert(destResource);
- return destPath;
- }
-
- /**
- * Copies the files matching {@code filter} at {@code src} path to {@code dest}
- *
- * @param src
- * @param dest
- * @param filter
- * @throws IOException
- */
- private void copyResourceFiles(Path src, Path dest, Predicate<Path> filter) throws IOException {
- try (Stream<Path> stream = Files.list(src)) {
- final List<Path> srcFiles = stream.filter(filter).collect(Collectors.toList());
- for (Path srcFile : srcFiles) {
- Path fileDest = Paths.get(dest.toString(), srcFile.getFileName().toString());
- Files.copy(srcFile, fileDest);
- }
- }
- }
-
- @Override
- public String toString() {
- return "{ \"class\" : \"" + getClass().getSimpleName() + "\" }";
- }
-}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index 77913ca..0526a32 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -40,12 +40,12 @@
import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
import org.apache.asterix.common.utils.PrintUtil;
import org.apache.asterix.common.utils.Servlets;
+import org.apache.asterix.common.utils.StorageConstants;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.messaging.MessagingChannelInterfaceFactory;
import org.apache.asterix.messaging.NCMessageBroker;
import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
import org.apache.asterix.util.MetadataBuiltinFunctions;
-import org.apache.asterix.utils.CompatibilityUtil;
import org.apache.hyracks.api.application.INCServiceContext;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.client.NodeStatus;
@@ -121,8 +121,10 @@
new MessagingChannelInterfaceFactory((NCMessageBroker) messageBroker, messagingProperties);
this.ncServiceCtx.setMessagingChannelInterfaceFactory(interfaceFactory);
final Checkpoint latestCheckpoint = runtimeContext.getTransactionSubsystem().getCheckpointManager().getLatest();
- if (latestCheckpoint != null) {
- CompatibilityUtil.ensureCompatibility(controllerService, latestCheckpoint.getStorageVersion());
+ if (latestCheckpoint != null && latestCheckpoint.getStorageVersion() != StorageConstants.VERSION) {
+ throw new IllegalStateException(
+ String.format("Storage version mismatch.. Current version (%s). On disk version: (%s)",
+ StorageConstants.VERSION, latestCheckpoint.getStorageVersion()));
}
if (LOGGER.isInfoEnabled()) {
IRecoveryManager recoveryMgr = runtimeContext.getTransactionSubsystem().getRecoveryManager();
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/CompatibilityUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/CompatibilityUtil.java
deleted file mode 100644
index fd9b1e8..0000000
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/CompatibilityUtil.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.utils;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.asterix.app.nc.task.MigrateStorageResourcesTask;
-import org.apache.asterix.common.api.INCLifecycleTask;
-import org.apache.asterix.common.utils.StorageConstants;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.control.nc.NodeControllerService;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-public class CompatibilityUtil {
-
- private static final Logger LOGGER = LogManager.getLogger();
- private static final int MIN_COMPATIBLE_VERSION = 1;
-
- private CompatibilityUtil() {
- }
-
- public static void ensureCompatibility(NodeControllerService ncs, int onDiskVerson) throws HyracksDataException {
- if (onDiskVerson == StorageConstants.VERSION) {
- return;
- }
- ensureUpgradability(onDiskVerson);
- LOGGER.info(() -> "Upgrading from storage version " + onDiskVerson + " to " + StorageConstants.VERSION);
- final List<INCLifecycleTask> upgradeTasks = getUpgradeTasks(onDiskVerson);
- for (INCLifecycleTask task : upgradeTasks) {
- task.perform(ncs);
- }
- }
-
- private static void ensureUpgradability(int onDiskVerson) {
- if (onDiskVerson < MIN_COMPATIBLE_VERSION) {
- throw new IllegalStateException(String.format(
- "Storage cannot be upgraded to new version. Current version (%s). On disk version: (%s)",
- StorageConstants.VERSION, onDiskVerson));
- }
- }
-
- private static List<INCLifecycleTask> getUpgradeTasks(int fromVersion) {
- List<INCLifecycleTask> upgradeTasks = new ArrayList<>();
- if (fromVersion < StorageConstants.REBALANCE_STORAGE_VERSION) {
- upgradeTasks.add(new MigrateStorageResourcesTask());
- }
- return upgradeTasks;
- }
-}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/MigrateStorageResourcesTaskTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/MigrateStorageResourcesTaskTest.java
deleted file mode 100644
index a98730a..0000000
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/MigrateStorageResourcesTaskTest.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.test.storage;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.function.Function;
-
-import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
-import org.apache.asterix.common.TestDataUtil;
-import org.apache.asterix.common.api.INcApplicationContext;
-import org.apache.asterix.common.config.GlobalConfig;
-import org.apache.asterix.common.storage.IndexPathElements;
-import org.apache.asterix.common.transactions.Checkpoint;
-import org.apache.asterix.common.utils.StorageConstants;
-import org.apache.asterix.common.utils.StoragePathUtil;
-import org.apache.asterix.transaction.management.service.recovery.AbstractCheckpointManager;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class MigrateStorageResourcesTaskTest {
-
- private static final String DEFAULT_TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
- private static final AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil();
-
- @Before
- public void setUp() throws Exception {
- }
-
- @After
- public void tearDown() throws Exception {
- integrationUtil.deinit(true);
- }
-
- @Test
- public void storageStructureMigration() throws Exception {
- Function<IndexPathElements, String> legacyIndexPathProvider = (
- pathElements) -> (pathElements.getRebalanceCount().equals("0") ? ""
- : pathElements.getRebalanceCount() + File.separator) + pathElements.getDatasetName()
- + StorageConstants.LEGACY_DATASET_INDEX_NAME_SEPARATOR + pathElements.getIndexName();
- StoragePathUtil.setIndexPathProvider(legacyIndexPathProvider);
- integrationUtil.init(true, DEFAULT_TEST_CONFIG_FILE_NAME);
- // create dataset and insert data using legacy structure
- String datasetName = "ds";
- TestDataUtil.createIdOnlyDataset(datasetName);
- TestDataUtil.upsertData(datasetName, 100);
- final long countBeforeMigration = TestDataUtil.getDatasetCount(datasetName);
- // stop NCs
- integrationUtil.deinit(false);
- // forge a checkpoint with old version to force migration to new storage structure on all ncs
- final INcApplicationContext nc1AppCtx = (INcApplicationContext) integrationUtil.ncs[0].getApplicationContext();
- final AbstractCheckpointManager nc1CheckpointManager = (AbstractCheckpointManager) nc1AppCtx
- .getTransactionSubsystem().getCheckpointManager();
- forgeOldVersionCheckpoint(nc1CheckpointManager);
- final INcApplicationContext nc2AppCtx = (INcApplicationContext) integrationUtil.ncs[1].getApplicationContext();
- final AbstractCheckpointManager nc2CheckpointManager = (AbstractCheckpointManager) nc2AppCtx
- .getTransactionSubsystem().getCheckpointManager();
- forgeOldVersionCheckpoint(nc2CheckpointManager);
-
- // remove the legacy path provider to use the new default structure
- StoragePathUtil.setIndexPathProvider(null);
- // start the NCs to do the migration
- integrationUtil.init(false, DEFAULT_TEST_CONFIG_FILE_NAME);
- final long countAfterMigration = TestDataUtil.getDatasetCount(datasetName);
- // ensure data migrated to new structure without issues
- Assert.assertEquals(countBeforeMigration, countAfterMigration);
- }
-
- private void forgeOldVersionCheckpoint(AbstractCheckpointManager manger) throws HyracksDataException {
- Checkpoint cp = new Checkpoint(-1, -1, 0, System.currentTimeMillis(), true,
- StorageConstants.REBALANCE_STORAGE_VERSION - 1);
- Path path = manger.getCheckpointPath(cp.getTimeStamp());
- // Write checkpoint file to disk
- try (BufferedWriter writer = Files.newBufferedWriter(path)) {
- writer.write(cp.asJson());
- writer.flush();
- } catch (IOException e) {
- throw HyracksDataException.create(e);
- }
- }
-}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/DatasetResourceReference.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/DatasetResourceReference.java
index c488b65..906a8b7 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/DatasetResourceReference.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/DatasetResourceReference.java
@@ -35,14 +35,6 @@
}
public static DatasetResourceReference of(LocalResource localResource) {
- return of(localResource, StorageConstants.VERSION);
- }
-
- public static DatasetResourceReference of(LocalResource localResource, int version) {
- if (version < StorageConstants.REBALANCE_STORAGE_VERSION) {
- // to support legacy storage migration
- return parseLegacyPath(localResource);
- }
return parse(localResource);
}
@@ -66,14 +58,6 @@
return datasetResourceReference;
}
- private static DatasetResourceReference parseLegacyPath(LocalResource localResource) {
- final DatasetResourceReference datasetResourceReference = new DatasetResourceReference();
- final String filePath = Paths.get(localResource.getPath(), StorageConstants.METADATA_FILE_NAME).toString();
- parseLegacyPath(datasetResourceReference, filePath);
- assignIds(localResource, datasetResourceReference);
- return datasetResourceReference;
- }
-
private static void assignIds(LocalResource localResource, DatasetResourceReference lrr) {
final DatasetLocalResource dsResource = (DatasetLocalResource) localResource.getResource();
lrr.datasetId = dsResource.getDatasetId();
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
index c7de7a3..52cf4a0 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
@@ -95,27 +95,6 @@
return Integer.parseInt(partition.substring(StorageConstants.PARTITION_DIR_PREFIX.length()));
}
- protected static void parseLegacyPath(ResourceReference ref, String path) {
- // old format: root/partition/dataverse/datasetName_idx_IndexName/fileName
- final String[] tokens = path.split(File.separator);
- if (tokens.length < 4) {
- throw new IllegalStateException("Unrecognized legacy path structure: " + path);
- }
- int offset = tokens.length;
- ref.name = tokens[--offset];
- // split combined dataset/index name
- final String[] indexTokens = tokens[--offset].split(StorageConstants.LEGACY_DATASET_INDEX_NAME_SEPARATOR);
- if (indexTokens.length != 2) {
- throw new IllegalStateException("Unrecognized legacy path structure: " + path);
- }
- ref.dataset = indexTokens[0];
- ref.index = indexTokens[1];
- ref.dataverse = tokens[--offset];
- ref.partition = tokens[--offset];
- ref.root = tokens[--offset];
- ref.rebalance = String.valueOf(0);
- }
-
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
index e83a75d..7b9a9a2 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
@@ -39,7 +39,6 @@
public class StoragePathUtil {
private static final Logger LOGGER = LogManager.getLogger();
- private static Function<IndexPathElements, String> indexPathProvider;
private StoragePathUtil() {
}
@@ -74,9 +73,6 @@
}
private static String prepareFullIndexName(String datasetName, String idxName, long rebalanceCount) {
- if (indexPathProvider != null) {
- return indexPathProvider.apply(new IndexPathElements(datasetName, idxName, String.valueOf(rebalanceCount)));
- }
return datasetName + File.separator + rebalanceCount + File.separator + idxName;
}
@@ -148,8 +144,4 @@
public static String getIndexNameFromPath(String path) {
return Paths.get(path).getFileName().toString();
}
-
- public static void setIndexPathProvider(Function<IndexPathElements, String> indexPathProvider) {
- StoragePathUtil.indexPathProvider = indexPathProvider;
- }
}
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index 71f6243..a1d0dd5 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -81,7 +81,6 @@
public class PersistentLocalResourceRepository implements ILocalResourceRepository {
- public static final Predicate<Path> INDEX_COMPONENTS = path -> !path.endsWith(StorageConstants.METADATA_FILE_NAME);
private static final Logger LOGGER = LogManager.getLogger();
private static final FilenameFilter LSM_INDEX_FILES_FILTER =
(dir, name) -> !name.startsWith(INDEX_CHECKPOINT_FILE_PREFIX);