001package org.apache.archiva.repository.storage; 002 003/* 004 * Licensed to the Apache Software Foundation (ASF) under one 005 * or more contributor license agreements. See the NOTICE file 006 * distributed with this work for additional information 007 * regarding copyright ownership. The ASF licenses this file 008 * to you under the Apache License, Version 2.0 (the 009 * "License"); you may not use this file except in compliance 010 * with the License. You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, 015 * software distributed under the License is distributed on an 016 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 017 * KIND, either express or implied. See the License for the 018 * specific language governing permissions and limitations 019 * under the License. 020 */ 021 022import org.apache.archiva.common.filelock.FileLockException; 023import org.apache.archiva.common.filelock.FileLockManager; 024import org.apache.archiva.common.filelock.FileLockTimeoutException; 025import org.apache.archiva.common.filelock.Lock; 026import org.apache.commons.lang3.StringUtils; 027import org.slf4j.Logger; 028import org.slf4j.LoggerFactory; 029 030import java.io.IOException; 031import java.nio.ByteBuffer; 032import java.nio.channels.FileChannel; 033import java.nio.channels.ReadableByteChannel; 034import java.nio.channels.WritableByteChannel; 035import java.nio.file.*; 036import java.util.HashSet; 037import java.util.function.Consumer; 038 039/** 040 * 041 * Utility class for assets. Allows to copy, move between different storage instances and 042 * recursively consume the tree. 043 * 044 * @author Martin Stockhammer <martin_s@apache.org> 045 */ 046public class StorageUtil 047{ 048 private static final int DEFAULT_BUFFER_SIZE = 4096; 049 private static final Logger log = LoggerFactory.getLogger(StorageUtil.class); 050 051 /** 052 * Copies the source asset to the target. The assets may be from different RepositoryStorage instances. 053 * If you know that source and asset are from the same storage instance, the copy method of the storage 054 * instance may be faster. 055 * 056 * @param source The source asset 057 * @param target The target asset 058 * @param locked If true, a readlock is set on the source and a write lock is set on the target. 059 * @param copyOptions Copy options 060 * @throws IOException 061 */ 062 public static final void copyAsset( final StorageAsset source, 063 final StorageAsset target, 064 boolean locked, 065 final CopyOption... copyOptions ) throws IOException 066 { 067 if (source.isFileBased() && target.isFileBased()) { 068 // Short cut for FS operations 069 final Path sourcePath = source.getFilePath(); 070 final Path targetPath = target.getFilePath( ); 071 if (locked) { 072 final FileLockManager lmSource = ((FilesystemStorage)source.getStorage()).getFileLockManager(); 073 final FileLockManager lmTarget = ((FilesystemStorage)target.getStorage()).getFileLockManager(); 074 Lock lockRead = null; 075 Lock lockWrite = null; 076 try { 077 lockRead = lmSource.readFileLock(sourcePath); 078 } catch (Exception e) { 079 log.error("Could not create read lock on {}", sourcePath); 080 throw new IOException(e); 081 } 082 try { 083 lockWrite = lmTarget.writeFileLock(targetPath); 084 } catch (Exception e) { 085 log.error("Could not create write lock on {}", targetPath); 086 throw new IOException(e); 087 } 088 try { 089 Files.copy(sourcePath, targetPath, copyOptions); 090 } finally { 091 if (lockRead!=null) { 092 try { 093 lmSource.release(lockRead); 094 } catch (FileLockException e) { 095 log.error("Error during lock release of read lock {}", lockRead.getFile()); 096 } 097 } 098 if (lockWrite!=null) { 099 try { 100 lmTarget.release(lockWrite); 101 } catch (FileLockException e) { 102 log.error("Error during lock release of write lock {}", lockWrite.getFile()); 103 } 104 } 105 } 106 } else 107 { 108 Files.copy( sourcePath, targetPath, copyOptions ); 109 } 110 } else { 111 try { 112 final RepositoryStorage sourceStorage = source.getStorage(); 113 final RepositoryStorage targetStorage = target.getStorage(); 114 sourceStorage.consumeDataFromChannel( source, is -> wrapWriteFunction( is, targetStorage, target, locked ), locked); 115 } catch (IOException e) { 116 throw e; 117 } catch (Throwable e) { 118 Throwable cause = e.getCause(); 119 if (cause instanceof IOException) { 120 throw (IOException)cause; 121 } else 122 { 123 throw new IOException( e ); 124 } 125 } 126 } 127 } 128 129 /** 130 * Moves a asset between different storage instances. 131 * If you know that source and asset are from the same storage instance, the move method of the storage 132 * instance may be faster. 133 * 134 * @param source The source asset 135 * @param target The target asset 136 * @param locked If true, a lock is used for the move operation. 137 * @param copyOptions Options for copying 138 * @throws IOException If the move fails 139 */ 140 public static final void moveAsset(StorageAsset source, StorageAsset target, boolean locked, CopyOption... copyOptions) throws IOException 141 { 142 if (source.isFileBased() && target.isFileBased()) { 143 // Short cut for FS operations 144 // Move is atomic operation 145 if (!Files.exists(target.getFilePath().getParent())) { 146 Files.createDirectories(target.getFilePath().getParent()); 147 } 148 Files.move( source.getFilePath(), target.getFilePath(), copyOptions ); 149 } else { 150 try { 151 final RepositoryStorage sourceStorage = source.getStorage(); 152 final RepositoryStorage targetStorage = target.getStorage(); 153 sourceStorage.consumeDataFromChannel( source, is -> wrapWriteFunction( is, targetStorage, target, locked ), locked); 154 sourceStorage.removeAsset( source ); 155 } catch (IOException e) { 156 throw e; 157 } catch (Throwable e) { 158 Throwable cause = e.getCause(); 159 if (cause instanceof IOException) { 160 throw (IOException)cause; 161 } else 162 { 163 throw new IOException( e ); 164 } 165 } 166 } 167 168 } 169 170 private static final void wrapWriteFunction(ReadableByteChannel is, RepositoryStorage targetStorage, StorageAsset target, boolean locked) { 171 try { 172 targetStorage.writeDataToChannel( target, os -> copy(is, os), locked ); 173 } catch (Exception e) { 174 throw new RuntimeException( e ); 175 } 176 } 177 178 179 private static final void copy( final ReadableByteChannel is, final WritableByteChannel os ) { 180 if (is instanceof FileChannel) { 181 copy( (FileChannel) is, os ); 182 } else if (os instanceof FileChannel) { 183 copy(is, (FileChannel)os); 184 } else 185 { 186 try 187 { 188 ByteBuffer buffer = ByteBuffer.allocate( DEFAULT_BUFFER_SIZE ); 189 while ( is.read( buffer ) != -1 ) 190 { 191 buffer.flip( ); 192 while ( buffer.hasRemaining( ) ) 193 { 194 os.write( buffer ); 195 } 196 buffer.clear( ); 197 } 198 } 199 catch ( IOException e ) 200 { 201 throw new RuntimeException( e ); 202 } 203 } 204 } 205 206 private static final void copy( final FileChannel is, final WritableByteChannel os ) { 207 try 208 { 209 is.transferTo( 0, is.size( ), os ); 210 } 211 catch ( IOException e ) 212 { 213 throw new RuntimeException( e ); 214 } 215 } 216 217 private static final void copy( final ReadableByteChannel is, final FileChannel os ) { 218 try 219 { 220 os.transferFrom( is, 0, Long.MAX_VALUE ); 221 } 222 catch ( IOException e ) 223 { 224 throw new RuntimeException( e ); 225 } 226 } 227 228 /** 229 * Runs the consumer function recursively on each asset found starting at the base path 230 * @param baseAsset The base path where to start search 231 * @param consumer The consumer function applied to each found asset 232 * @param depthFirst If true, the deepest elements are consumed first. 233 * @param maxDepth The maximum depth to recurse into. 0 means, only the baseAsset is consumed, 1 the base asset and its children and so forth. 234 */ 235 public static final void recurse(final StorageAsset baseAsset, final Consumer<StorageAsset> consumer, final boolean depthFirst, final int maxDepth) throws IOException { 236 recurse(baseAsset, consumer, depthFirst, maxDepth, 0); 237 } 238 239 /** 240 * Runs the consumer function recursively on each asset found starting at the base path. The function descends into 241 * maximum depth. 242 * 243 * @param baseAsset The base path where to start search 244 * @param consumer The consumer function applied to each found asset 245 * @param depthFirst If true, the deepest elements are consumed first. 246 */ 247 public static final void recurse(final StorageAsset baseAsset, final Consumer<StorageAsset> consumer, final boolean depthFirst) throws IOException { 248 recurse(baseAsset, consumer, depthFirst, Integer.MAX_VALUE, 0); 249 } 250 251 /** 252 * Runs the consumer function recursively on each asset found starting at the base path. It does not recurse with 253 * depth first and stops only if there are no more children available. 254 * 255 * @param baseAsset The base path where to start search 256 * @param consumer The consumer function applied to each found asset 257 */ 258 public static final void recurse(final StorageAsset baseAsset, final Consumer<StorageAsset> consumer) throws IOException { 259 recurse(baseAsset, consumer, false, Integer.MAX_VALUE, 0); 260 } 261 262 private static final void recurse(final StorageAsset baseAsset, final Consumer<StorageAsset> consumer, final boolean depthFirst, final int maxDepth, final int currentDepth) 263 throws IOException { 264 if (!depthFirst) { 265 consumer.accept(baseAsset); 266 } 267 if (currentDepth<maxDepth && baseAsset.isContainer()) { 268 for(StorageAsset asset : baseAsset.list() ) { 269 recurse(asset, consumer, depthFirst, maxDepth, currentDepth+1); 270 } 271 } 272 if (depthFirst) { 273 consumer.accept(baseAsset); 274 } 275 } 276 277 /** 278 * Deletes the given asset and all child assets recursively. 279 * @param baseDir The base asset to remove. 280 * @throws IOException 281 */ 282 public static final void deleteRecursively(StorageAsset baseDir) throws IOException { 283 recurse(baseDir, a -> { 284 try { 285 a.getStorage().removeAsset(a); 286 } catch (IOException e) { 287 log.error("Could not delete asset {}", a.getPath()); 288 } 289 },true); 290 } 291 292 /** 293 * Returns the extension of the name of a given asset. Extension is the substring after the last occurence of '.' in the 294 * string. If no '.' is found, the empty string is returned. 295 * 296 * @param asset The asset from which to return the extension string. 297 * @return The extension. 298 */ 299 public static final String getExtension(StorageAsset asset) { 300 return StringUtils.substringAfterLast(asset.getName(),"."); 301 } 302 303 public static final void copyToLocalFile(StorageAsset asset, Path destination, CopyOption... copyOptions) throws IOException { 304 if (asset.isFileBased()) { 305 Files.copy(asset.getFilePath(), destination, copyOptions); 306 } else { 307 try { 308 309 HashSet<OpenOption> openOptions = new HashSet<>(); 310 for (CopyOption option : copyOptions) { 311 if (option == StandardCopyOption.REPLACE_EXISTING) { 312 openOptions.add(StandardOpenOption.CREATE); 313 openOptions.add(StandardOpenOption.TRUNCATE_EXISTING); 314 openOptions.add(StandardOpenOption.WRITE); 315 } else { 316 openOptions.add(StandardOpenOption.WRITE); 317 openOptions.add(StandardOpenOption.CREATE_NEW); 318 } 319 } 320 asset.getStorage().consumeDataFromChannel(asset, channel -> { 321 try { 322 FileChannel.open(destination, openOptions).transferFrom(channel, 0, Long.MAX_VALUE); 323 } catch (IOException e) { 324 throw new RuntimeException(e); 325 } 326 }, false); 327 } catch (Throwable e) { 328 if (e.getCause() instanceof IOException) { 329 throw (IOException)e.getCause(); 330 } else { 331 throw new IOException(e); 332 } 333 } 334 } 335 } 336 337 public static class PathInformation { 338 final Path path ; 339 final boolean tmpFile; 340 341 PathInformation(Path path, boolean tmpFile) { 342 this.path = path; 343 this.tmpFile = tmpFile; 344 } 345 346 public Path getPath() { 347 return path; 348 } 349 350 public boolean isTmpFile() { 351 return tmpFile; 352 } 353 354 } 355 356 public static final PathInformation getAssetDataAsPath(StorageAsset asset) throws IOException { 357 if (!asset.exists()) { 358 throw new IOException("Asset does not exist"); 359 } 360 if (asset.isFileBased()) { 361 return new PathInformation(asset.getFilePath(), false); 362 } else { 363 Path tmpFile = Files.createTempFile(asset.getName(), getExtension(asset)); 364 copyToLocalFile(asset, tmpFile, StandardCopyOption.REPLACE_EXISTING); 365 return new PathInformation(tmpFile, true); 366 } 367 } 368 369}