001package org.apache.archiva.scheduler.indexing; 002/* 003 * Licensed to the Apache Software Foundation (ASF) under one 004 * or more contributor license agreements. See the NOTICE file 005 * distributed with this work for additional information 006 * regarding copyright ownership. The ASF licenses this file 007 * to you under the Apache License, Version 2.0 (the 008 * "License"); you may not use this file except in compliance 009 * with the License. You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, 014 * software distributed under the License is distributed on an 015 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 016 * KIND, either express or implied. See the License for the 017 * specific language governing permissions and limitations 018 * under the License. 019 */ 020 021import org.apache.archiva.admin.model.RepositoryAdminException; 022import org.apache.archiva.admin.model.beans.NetworkProxy; 023import org.apache.archiva.admin.model.beans.RemoteRepository; 024import org.apache.archiva.admin.model.remote.RemoteRepositoryAdmin; 025import org.apache.archiva.proxy.common.WagonFactory; 026import org.apache.archiva.proxy.common.WagonFactoryException; 027import org.apache.archiva.proxy.common.WagonFactoryRequest; 028import org.apache.commons.io.FileUtils; 029import org.apache.commons.lang.time.StopWatch; 030import org.apache.maven.index.context.IndexingContext; 031import org.apache.maven.index.updater.IndexUpdateRequest; 032import org.apache.maven.index.updater.IndexUpdater; 033import org.apache.maven.index.updater.ResourceFetcher; 034import org.apache.maven.wagon.ConnectionException; 035import org.apache.maven.wagon.ResourceDoesNotExistException; 036import org.apache.maven.wagon.StreamWagon; 037import org.apache.maven.wagon.TransferFailedException; 038import org.apache.maven.wagon.Wagon; 039import org.apache.maven.wagon.authentication.AuthenticationException; 040import org.apache.maven.wagon.authentication.AuthenticationInfo; 041import org.apache.maven.wagon.authorization.AuthorizationException; 042import org.apache.maven.wagon.events.TransferEvent; 043import org.apache.maven.wagon.events.TransferListener; 044import org.apache.maven.wagon.providers.http.AbstractHttpClientWagon; 045import org.apache.maven.wagon.providers.http.HttpConfiguration; 046import org.apache.maven.wagon.providers.http.HttpMethodConfiguration; 047import org.apache.maven.wagon.proxy.ProxyInfo; 048import org.apache.maven.wagon.repository.Repository; 049import org.slf4j.Logger; 050import org.slf4j.LoggerFactory; 051 052import java.io.File; 053import java.io.FileNotFoundException; 054import java.io.IOException; 055import java.io.InputStream; 056import java.net.MalformedURLException; 057import java.net.URL; 058import java.nio.file.Files; 059import java.util.List; 060import java.util.Map; 061 062/** 063 * @author Olivier Lamy 064 * @since 1.4-M1 065 */ 066public class DownloadRemoteIndexTask 067 implements Runnable 068{ 069 private Logger log = LoggerFactory.getLogger( getClass() ); 070 071 private RemoteRepository remoteRepository; 072 073 private RemoteRepositoryAdmin remoteRepositoryAdmin; 074 075 private WagonFactory wagonFactory; 076 077 private NetworkProxy networkProxy; 078 079 private boolean fullDownload; 080 081 private List<String> runningRemoteDownloadIds; 082 083 private IndexUpdater indexUpdater; 084 085 086 public DownloadRemoteIndexTask( DownloadRemoteIndexTaskRequest downloadRemoteIndexTaskRequest, 087 List<String> runningRemoteDownloadIds ) 088 { 089 this.remoteRepository = downloadRemoteIndexTaskRequest.getRemoteRepository(); 090 this.wagonFactory = downloadRemoteIndexTaskRequest.getWagonFactory(); 091 this.networkProxy = downloadRemoteIndexTaskRequest.getNetworkProxy(); 092 this.fullDownload = downloadRemoteIndexTaskRequest.isFullDownload(); 093 this.runningRemoteDownloadIds = runningRemoteDownloadIds; 094 this.indexUpdater = downloadRemoteIndexTaskRequest.getIndexUpdater(); 095 this.remoteRepositoryAdmin = downloadRemoteIndexTaskRequest.getRemoteRepositoryAdmin(); 096 } 097 098 @Override 099 public void run() 100 { 101 102 // so short lock : not sure we need it 103 synchronized ( this.runningRemoteDownloadIds ) 104 { 105 if ( this.runningRemoteDownloadIds.contains( this.remoteRepository.getId() ) ) 106 { 107 // skip it as it's running 108 log.info( "skip download index remote for repo {} it's already running", 109 this.remoteRepository.getId() ); 110 return; 111 } 112 this.runningRemoteDownloadIds.add( this.remoteRepository.getId() ); 113 } 114 File tempIndexDirectory = null; 115 StopWatch stopWatch = new StopWatch(); 116 stopWatch.start(); 117 try 118 { 119 log.info( "start download remote index for remote repository {}", this.remoteRepository.getId() ); 120 IndexingContext indexingContext = remoteRepositoryAdmin.createIndexContext( this.remoteRepository ); 121 122 // create a temp directory to download files 123 tempIndexDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".tmpIndex" ); 124 File indexCacheDirectory = new File( indexingContext.getIndexDirectoryFile().getParent(), ".indexCache" ); 125 indexCacheDirectory.mkdirs(); 126 if ( tempIndexDirectory.exists() ) 127 { 128 FileUtils.deleteDirectory( tempIndexDirectory ); 129 } 130 tempIndexDirectory.mkdirs(); 131 tempIndexDirectory.deleteOnExit(); 132 String baseIndexUrl = indexingContext.getIndexUpdateUrl(); 133 134 String wagonProtocol = new URL( this.remoteRepository.getUrl() ).getProtocol(); 135 136 final StreamWagon wagon = (StreamWagon) wagonFactory.getWagon( 137 new WagonFactoryRequest( wagonProtocol, this.remoteRepository.getExtraHeaders() ).networkProxy( 138 this.networkProxy ) 139 ); 140 // FIXME olamy having 2 config values 141 wagon.setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 ); 142 wagon.setTimeout( remoteRepository.getTimeout() * 1000 ); 143 144 if ( wagon instanceof AbstractHttpClientWagon ) 145 { 146 HttpConfiguration httpConfiguration = new HttpConfiguration(); 147 HttpMethodConfiguration httpMethodConfiguration = new HttpMethodConfiguration(); 148 httpMethodConfiguration.setUsePreemptive( true ); 149 httpMethodConfiguration.setReadTimeout( remoteRepository.getRemoteDownloadTimeout() * 1000 ); 150 httpConfiguration.setGet( httpMethodConfiguration ); 151 AbstractHttpClientWagon.class.cast( wagon ).setHttpConfiguration( httpConfiguration ); 152 } 153 154 wagon.addTransferListener( new DownloadListener() ); 155 ProxyInfo proxyInfo = null; 156 if ( this.networkProxy != null ) 157 { 158 proxyInfo = new ProxyInfo(); 159 proxyInfo.setType( this.networkProxy.getProtocol() ); 160 proxyInfo.setHost( this.networkProxy.getHost() ); 161 proxyInfo.setPort( this.networkProxy.getPort() ); 162 proxyInfo.setUserName( this.networkProxy.getUsername() ); 163 proxyInfo.setPassword( this.networkProxy.getPassword() ); 164 } 165 AuthenticationInfo authenticationInfo = null; 166 if ( this.remoteRepository.getUserName() != null ) 167 { 168 authenticationInfo = new AuthenticationInfo(); 169 authenticationInfo.setUserName( this.remoteRepository.getUserName() ); 170 authenticationInfo.setPassword( this.remoteRepository.getPassword() ); 171 } 172 wagon.connect( new Repository( this.remoteRepository.getId(), baseIndexUrl ), authenticationInfo, 173 proxyInfo ); 174 175 File indexDirectory = indexingContext.getIndexDirectoryFile(); 176 if ( !indexDirectory.exists() ) 177 { 178 indexDirectory.mkdirs(); 179 } 180 181 ResourceFetcher resourceFetcher = 182 new WagonResourceFetcher( log, tempIndexDirectory, wagon, remoteRepository ); 183 IndexUpdateRequest request = new IndexUpdateRequest( indexingContext, resourceFetcher ); 184 request.setForceFullUpdate( this.fullDownload ); 185 request.setLocalIndexCacheDir( indexCacheDirectory ); 186 187 this.indexUpdater.fetchAndUpdateIndex( request ); 188 stopWatch.stop(); 189 log.info( "time update index from remote for repository {}: {} s", this.remoteRepository.getId(), 190 ( stopWatch.getTime() / 1000 ) ); 191 192 // index packing optionnal ?? 193 //IndexPackingRequest indexPackingRequest = 194 // new IndexPackingRequest( indexingContext, indexingContext.getIndexDirectoryFile() ); 195 //indexPacker.packIndex( indexPackingRequest ); 196 indexingContext.updateTimestamp( true ); 197 198 } 199 catch ( MalformedURLException e ) 200 { 201 log.error( e.getMessage(), e ); 202 throw new RuntimeException( e.getMessage(), e ); 203 } 204 catch ( WagonFactoryException e ) 205 { 206 log.error( e.getMessage(), e ); 207 throw new RuntimeException( e.getMessage(), e ); 208 } 209 catch ( ConnectionException e ) 210 { 211 log.error( e.getMessage(), e ); 212 throw new RuntimeException( e.getMessage(), e ); 213 } 214 catch ( AuthenticationException e ) 215 { 216 log.error( e.getMessage(), e ); 217 throw new RuntimeException( e.getMessage(), e ); 218 } 219 catch ( IOException e ) 220 { 221 log.error( e.getMessage(), e ); 222 throw new RuntimeException( e.getMessage(), e ); 223 } 224 catch ( RepositoryAdminException e ) 225 { 226 log.error( e.getMessage(), e ); 227 throw new RuntimeException( e.getMessage(), e ); 228 } 229 finally 230 { 231 deleteDirectoryQuiet( tempIndexDirectory ); 232 this.runningRemoteDownloadIds.remove( this.remoteRepository.getId() ); 233 } 234 log.info( "end download remote index for remote repository {}", this.remoteRepository.getId() ); 235 } 236 237 private void deleteDirectoryQuiet( File f ) 238 { 239 try 240 { 241 FileUtils.deleteDirectory( f ); 242 } 243 catch ( IOException e ) 244 { 245 log.warn( "skip error delete {} : {}", f, e.getMessage() ); 246 } 247 } 248 249 250 private static final class DownloadListener 251 implements TransferListener 252 { 253 private Logger log = LoggerFactory.getLogger( getClass() ); 254 255 private String resourceName; 256 257 private long startTime; 258 259 private int totalLength = 0; 260 261 @Override 262 public void transferInitiated( TransferEvent transferEvent ) 263 { 264 startTime = System.currentTimeMillis(); 265 resourceName = transferEvent.getResource().getName(); 266 log.debug( "initiate transfer of {}", resourceName ); 267 } 268 269 @Override 270 public void transferStarted( TransferEvent transferEvent ) 271 { 272 this.totalLength = 0; 273 resourceName = transferEvent.getResource().getName(); 274 log.info( "start transfer of {}", transferEvent.getResource().getName() ); 275 } 276 277 @Override 278 public void transferProgress( TransferEvent transferEvent, byte[] buffer, int length ) 279 { 280 log.debug( "transfer of {} : {}/{}", transferEvent.getResource().getName(), buffer.length, length ); 281 this.totalLength += length; 282 } 283 284 @Override 285 public void transferCompleted( TransferEvent transferEvent ) 286 { 287 resourceName = transferEvent.getResource().getName(); 288 long endTime = System.currentTimeMillis(); 289 log.info( "end of transfer file {} {} kb: {}s", transferEvent.getResource().getName(), 290 this.totalLength / 1024, ( endTime - startTime ) / 1000 ); 291 } 292 293 @Override 294 public void transferError( TransferEvent transferEvent ) 295 { 296 log.info( "error of transfer file {}: {}", transferEvent.getResource().getName(), 297 transferEvent.getException().getMessage(), transferEvent.getException() ); 298 } 299 300 @Override 301 public void debug( String message ) 302 { 303 log.debug( "transfer debug {}", message ); 304 } 305 } 306 307 private static class WagonResourceFetcher 308 implements ResourceFetcher 309 { 310 311 Logger log; 312 313 File tempIndexDirectory; 314 315 Wagon wagon; 316 317 RemoteRepository remoteRepository; 318 319 private WagonResourceFetcher( Logger log, File tempIndexDirectory, Wagon wagon, 320 RemoteRepository remoteRepository ) 321 { 322 this.log = log; 323 this.tempIndexDirectory = tempIndexDirectory; 324 this.wagon = wagon; 325 this.remoteRepository = remoteRepository; 326 } 327 328 @Override 329 public void connect( String id, String url ) 330 throws IOException 331 { 332 //no op 333 } 334 335 @Override 336 public void disconnect() 337 throws IOException 338 { 339 // no op 340 } 341 342 @Override 343 public InputStream retrieve( String name ) 344 throws IOException, FileNotFoundException 345 { 346 try 347 { 348 log.info( "index update retrieve file, name:{}", name ); 349 File file = new File( tempIndexDirectory, name ); 350 Files.deleteIfExists( file.toPath() ); 351 file.deleteOnExit(); 352 wagon.get( addParameters( name, this.remoteRepository ), file ); 353 return Files.newInputStream( file.toPath() ); 354 } 355 catch ( AuthorizationException | TransferFailedException e ) 356 { 357 throw new IOException( e.getMessage(), e ); 358 } 359 catch ( ResourceDoesNotExistException e ) 360 { 361 FileNotFoundException fnfe = new FileNotFoundException( e.getMessage() ); 362 fnfe.initCause( e ); 363 throw fnfe; 364 } 365 } 366 367 // FIXME remove crappy copy/paste 368 protected String addParameters( String path, RemoteRepository remoteRepository ) 369 { 370 if ( remoteRepository.getExtraParameters().isEmpty() ) 371 { 372 return path; 373 } 374 375 boolean question = false; 376 377 StringBuilder res = new StringBuilder( path == null ? "" : path ); 378 379 for ( Map.Entry<String, String> entry : remoteRepository.getExtraParameters().entrySet() ) 380 { 381 if ( !question ) 382 { 383 res.append( '?' ).append( entry.getKey() ).append( '=' ).append( entry.getValue() ); 384 } 385 } 386 387 return res.toString(); 388 } 389 390 } 391 392 393} 394