001package org.apache.archiva.scheduler.indexing.maven; 002 003/* 004 * Licensed to the Apache Software Foundation (ASF) under one 005 * or more contributor license agreements. See the NOTICE file 006 * distributed with this work for additional information 007 * regarding copyright ownership. The ASF licenses this file 008 * to you under the Apache License, Version 2.0 (the 009 * "License"); you may not use this file except in compliance 010 * with the License. You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, 015 * software distributed under the Li 016 * cense is distributed on an 017 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 018 * KIND, either express or implied. See the License for the 019 * specific language governing permissions and limitations 020 * under the License. 021 */ 022 023import org.apache.archiva.indexer.ArchivaIndexingContext; 024import org.apache.archiva.indexer.UnsupportedBaseContextException; 025import org.apache.archiva.components.taskqueue.Task; 026import org.apache.archiva.components.taskqueue.execution.TaskExecutionException; 027import org.apache.archiva.components.taskqueue.execution.TaskExecutor; 028import org.apache.archiva.repository.ManagedRepository; 029import org.apache.archiva.repository.features.IndexCreationFeature; 030import org.apache.archiva.scheduler.indexing.ArtifactIndexingTask; 031import org.apache.maven.index.ArtifactContext; 032import org.apache.maven.index.ArtifactContextProducer; 033import org.apache.maven.index.DefaultScannerListener; 034import org.apache.maven.index.FlatSearchRequest; 035import org.apache.maven.index.FlatSearchResponse; 036import org.apache.maven.index.Indexer; 037import org.apache.maven.index.IndexerEngine; 038import org.apache.maven.index.MAVEN; 039import org.apache.maven.index.Scanner; 040import org.apache.maven.index.ScanningRequest; 041import org.apache.maven.index.ScanningResult; 042import org.apache.maven.index.context.IndexingContext; 043import org.apache.maven.index.expr.SourcedSearchExpression; 044import org.apache.maven.index.packer.IndexPacker; 045import org.apache.maven.index.packer.IndexPackingRequest; 046import org.apache.maven.index_shaded.lucene.search.BooleanClause; 047import org.apache.maven.index_shaded.lucene.search.BooleanQuery; 048import org.slf4j.Logger; 049import org.slf4j.LoggerFactory; 050import org.springframework.stereotype.Service; 051 052import javax.inject.Inject; 053import java.io.IOException; 054import java.nio.file.Path; 055 056/** 057 * ArchivaIndexingTaskExecutor Executes all indexing tasks. Adding, updating and removing artifacts from the index are 058 * all performed by this executor. Add and update artifact in index tasks are added in the indexing task queue by the 059 * NexusIndexerConsumer while remove artifact from index tasks are added by the LuceneCleanupRemoveIndexedConsumer. 060 */ 061@Service( "taskExecutor#indexing" ) 062public class ArchivaIndexingTaskExecutor 063 implements TaskExecutor 064{ 065 private Logger log = LoggerFactory.getLogger( ArchivaIndexingTaskExecutor.class ); 066 067 @Inject 068 private IndexPacker indexPacker; 069 070 @Inject 071 private ArtifactContextProducer artifactContextProducer; 072 073 @Inject 074 private Indexer indexer; 075 076 @Inject 077 private Scanner scanner; 078 079 @Inject 080 IndexerEngine indexerEngine; 081 082 /** 083 * depending on current {@link Task} you have. 084 * If {@link org.apache.archiva.scheduler.indexing.ArtifactIndexingTask.Action#FINISH} && isExecuteOnEntireRepo: 085 * repository will be scanned. 086 * 087 * @param task 088 * @throws TaskExecutionException 089 */ 090 @Override 091 public void executeTask( Task task ) 092 throws TaskExecutionException 093 { 094 ArtifactIndexingTask indexingTask = (ArtifactIndexingTask) task; 095 096 ManagedRepository repository = indexingTask.getRepository( ); 097 ArchivaIndexingContext archivaContext = indexingTask.getContext( ); 098 IndexingContext context = null; 099 try 100 { 101 context = archivaContext.getBaseContext( IndexingContext.class ); 102 } 103 catch ( UnsupportedBaseContextException e ) 104 { 105 throw new TaskExecutionException( "Bad repository type.", e ); 106 } 107 108 if ( ArtifactIndexingTask.Action.FINISH.equals( indexingTask.getAction( ) ) 109 && indexingTask.isExecuteOnEntireRepo( ) ) 110 { 111 long start = System.currentTimeMillis( ); 112 try 113 { 114 context.updateTimestamp( ); 115 DefaultScannerListener listener = new DefaultScannerListener( context, indexerEngine, true, null ); 116 ScanningRequest request = new ScanningRequest( context, listener ); 117 ScanningResult result = scanner.scan( request ); 118 if ( result.hasExceptions( ) ) 119 { 120 log.error( "Exceptions occured during index scan of " + context.getId( ) ); 121 result.getExceptions( ).stream( ).map( e -> e.getMessage( ) ).distinct( ).limit( 5 ).forEach( 122 s -> log.error( "Message: " + s ) 123 ); 124 } 125 } 126 catch ( IOException e ) 127 { 128 log.error( "Error during context scan {}: {}", context.getId( ), context.getIndexDirectory( ) ); 129 } 130 long end = System.currentTimeMillis( ); 131 log.info( "indexed maven repository: {}, onlyUpdate: {}, time {} ms", repository.getId( ), 132 indexingTask.isOnlyUpdate( ), ( end - start ) ); 133 log.debug( "Finishing indexing task on repo: {}", repository.getId( ) ); 134 finishIndexingTask( indexingTask, repository, context ); 135 } 136 else 137 { 138 // create context if not a repo scan request 139 if ( !indexingTask.isExecuteOnEntireRepo( ) ) 140 { 141 try 142 { 143 log.debug( "Creating indexing context on resource: {}", // 144 ( indexingTask.getResourceFile( ) == null 145 ? "none" 146 : indexingTask.getResourceFile( ) ) ); 147 archivaContext = repository.getIndexingContext( ); 148 context = archivaContext.getBaseContext( IndexingContext.class ); 149 } 150 catch ( UnsupportedBaseContextException e ) 151 { 152 log.error( "Error occurred while creating context: {}", e.getMessage( ) ); 153 throw new TaskExecutionException( "Error occurred while creating context: " + e.getMessage( ), e ); 154 } 155 } 156 157 if ( context == null || context.getIndexDirectory( ) == null ) 158 { 159 throw new TaskExecutionException( "Trying to index an artifact but the context is already closed" ); 160 } 161 162 try 163 { 164 Path artifactFile = indexingTask.getResourceFile( ); 165 if ( artifactFile == null ) 166 { 167 log.debug( "no artifact pass in indexing task so skip it" ); 168 } 169 else 170 { 171 ArtifactContext ac = artifactContextProducer.getArtifactContext( context, artifactFile.toFile( ) ); 172 173 if ( ac != null ) 174 { 175 // MRM-1779 pom must be indexed too 176 // TODO make that configurable? 177 if ( artifactFile.getFileName( ).toString( ).endsWith( ".pom" ) ) 178 { 179 ac.getArtifactInfo( ).setFileExtension( "pom" ); 180 ac.getArtifactInfo( ).setPackaging( "pom" ); 181 ac.getArtifactInfo( ).setClassifier( "pom" ); 182 } 183 if ( indexingTask.getAction( ).equals( ArtifactIndexingTask.Action.ADD ) ) 184 { 185 //IndexSearcher s = context.getIndexSearcher(); 186 //String uinfo = ac.getArtifactInfo().getUinfo(); 187 //TopDocs d = s.search( new TermQuery( new Term( ArtifactInfo.UINFO, uinfo ) ), 1 ); 188 189 BooleanQuery.Builder qb = new BooleanQuery.Builder(); 190 qb.add( indexer.constructQuery( MAVEN.GROUP_ID, new SourcedSearchExpression( 191 ac.getArtifactInfo( ).getGroupId( ) ) ), BooleanClause.Occur.MUST ); 192 qb.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new SourcedSearchExpression( 193 ac.getArtifactInfo( ).getArtifactId( ) ) ), BooleanClause.Occur.MUST ); 194 qb.add( indexer.constructQuery( MAVEN.VERSION, new SourcedSearchExpression( 195 ac.getArtifactInfo( ).getVersion( ) ) ), BooleanClause.Occur.MUST ); 196 if ( ac.getArtifactInfo( ).getClassifier( ) != null ) 197 { 198 qb.add( indexer.constructQuery( MAVEN.CLASSIFIER, new SourcedSearchExpression( 199 ac.getArtifactInfo( ).getClassifier( ) ) ), BooleanClause.Occur.MUST ); 200 } 201 if ( ac.getArtifactInfo( ).getPackaging( ) != null ) 202 { 203 qb.add( indexer.constructQuery( MAVEN.PACKAGING, new SourcedSearchExpression( 204 ac.getArtifactInfo( ).getPackaging( ) ) ), BooleanClause.Occur.MUST ); 205 } 206 FlatSearchRequest flatSearchRequest = new FlatSearchRequest( qb.build(), context ); 207 FlatSearchResponse flatSearchResponse = indexer.searchFlat( flatSearchRequest ); 208 if ( flatSearchResponse.getResults( ).isEmpty( ) ) 209 { 210 log.debug( "Adding artifact '{}' to index..", ac.getArtifactInfo( ) ); 211 indexerEngine.index( context, ac ); 212 } 213 else 214 { 215 log.debug( "Updating artifact '{}' in index..", ac.getArtifactInfo( ) ); 216 // TODO check if update exists !! 217 indexerEngine.update( context, ac ); 218 } 219 220 context.updateTimestamp( ); 221 context.commit( ); 222 223 224 } 225 else 226 { 227 log.debug( "Removing artifact '{}' from index..", ac.getArtifactInfo( ) ); 228 indexerEngine.remove( context, ac ); 229 } 230 } 231 } 232 // close the context if not a repo scan request 233 if ( !indexingTask.isExecuteOnEntireRepo( ) ) 234 { 235 log.debug( "Finishing indexing task on resource file : {}", indexingTask.getResourceFile( ) != null 236 ? indexingTask.getResourceFile( ) 237 : " none " ); 238 finishIndexingTask( indexingTask, repository, context ); 239 } 240 } 241 catch ( IOException e ) 242 { 243 log.error( "Error occurred while executing indexing task '{}': {}", indexingTask, e.getMessage( ), 244 e ); 245 throw new TaskExecutionException( "Error occurred while executing indexing task '" + indexingTask + "'", 246 e ); 247 } 248 } 249 250 } 251 252 private void finishIndexingTask( ArtifactIndexingTask indexingTask, ManagedRepository repository, 253 IndexingContext context ) 254 throws TaskExecutionException 255 { 256 try 257 { 258 259 log.debug( "Finishing indexing" ); 260 context.optimize( ); 261 262 if ( repository.supportsFeature( IndexCreationFeature.class ) ) 263 { 264 IndexCreationFeature icf = repository.getFeature( IndexCreationFeature.class ).get( ); 265 if ( !icf.isSkipPackedIndexCreation( ) && icf.getLocalPackedIndexPath( ) != null && icf.getLocalIndexPath().getFilePath()!=null ) 266 { 267 268 log.debug( "Creating packed index from {} on {}", context.getIndexDirectoryFile( ), icf.getLocalPackedIndexPath( ) ); 269 IndexPackingRequest request = new IndexPackingRequest( context, // 270 context.acquireIndexSearcher( ).getIndexReader( ), 271 // 272 icf.getLocalPackedIndexPath( ).getFilePath().toFile( ) ); 273 274 indexPacker.packIndex( request ); 275 context.updateTimestamp( true ); 276 277 log.debug( "Index file packed at '{}'.", icf.getLocalPackedIndexPath( ) ); 278 } 279 else 280 { 281 log.debug( "skip packed index creation" ); 282 } 283 } 284 else 285 { 286 log.debug( "skip packed index creation" ); 287 } 288 } 289 catch ( IOException e ) 290 { 291 log.error( "Error occurred while executing indexing task '{}': {}", indexingTask, e.getMessage( ) ); 292 throw new TaskExecutionException( "Error occurred while executing indexing task '" + indexingTask + "'", 293 e ); 294 } 295 } 296 297 public void setIndexPacker( IndexPacker indexPacker ) 298 { 299 this.indexPacker = indexPacker; 300 } 301 302}