最近很多小伙伴都在问com.facebook.presto.client.FailureInfo的实例源码和facebook源代码这两个问题,那么本篇文章就来给大家详细解答一下,同时本文还将给你拓展c
最近很多小伙伴都在问com.facebook.presto.client.FailureInfo的实例源码和facebook源代码这两个问题,那么本篇文章就来给大家详细解答一下,同时本文还将给你拓展com.facebook.presto.client.ClientSession的实例源码、com.facebook.presto.client.Column的实例源码、com.facebook.presto.client.QueryError的实例源码、com.facebook.presto.client.QueryResults的实例源码等相关知识,下面开始了哦!
本文目录一览:- com.facebook.presto.client.FailureInfo的实例源码(facebook源代码)
- com.facebook.presto.client.ClientSession的实例源码
- com.facebook.presto.client.Column的实例源码
- com.facebook.presto.client.QueryError的实例源码
- com.facebook.presto.client.QueryResults的实例源码
com.facebook.presto.client.FailureInfo的实例源码(facebook源代码)
@Description("Decodes json to an exception and throws it") @ScalarFunction @sqlType("unkNown") public static void fail(@sqlType(StandardTypes.JSON) Slice failureInfoSlice) { FailureInfo failureInfo = JSON_CODEC.fromJson(failureInfoSlice.getBytes()); // wrap the failure in a new exception to append the current stack trace throw new PrestoException(StandardErrorCode.USER_ERROR,failureInfo.toException()); }
public FailureInfo toFailureInfo() { List<FailureInfo> suppressed = this.suppressed.stream() .map(ExecutionFailureInfo::toFailureInfo) .collect(toImmutableList()); return new FailureInfo(type,message,cause == null ? null : cause.toFailureInfo(),suppressed,stack,errorLocation); }
private static QueryError toQueryError(QueryInfo queryInfo) { FailureInfo failure = queryInfo.getFailureInfo(); if (failure == null) { QueryState state = queryInfo.getState(); if ((!state.isDone()) || (state == QueryState.FINISHED)) { return null; } log.warn("Query %s in state %s has no failure info",queryInfo.getQueryId(),state); failure = toFailure(new RuntimeException(format("Query is %s (reason unkNown)",state))).toFailureInfo(); } ErrorCode errorCode; if (queryInfo.getErrorCode() != null) { errorCode = queryInfo.getErrorCode(); } else { errorCode = INTERNAL_ERROR.toErrorCode(); log.warn("Failed query %s has no error code",queryInfo.getQueryId()); } return new QueryError( failure.getMessage(),null,errorCode.getCode(),errorCode.getName(),toErrorType(errorCode.getCode()).toString(),failure.getErrorLocation(),failure); }
@VisibleForTesting @NotNull public static Expression createFailureFunction(RuntimeException exception,Type type) { requireNonNull(exception,"Exception is null"); String failureInfo = JsonCodec.jsonCodec(FailureInfo.class).toJson(Failures.toFailure(exception).toFailureInfo()); FunctionCall jsonParse = new FunctionCall(Qualifiedname.of("json_parse"),ImmutableList.of(new StringLiteral(failureInfo))); FunctionCall failureFunction = new FunctionCall(Qualifiedname.of("fail"),ImmutableList.of(jsonParse)); return new Cast(failureFunction,type.getTypeSignature().toString()); }
@Override public TaskInfo getTaskInfo() { TaskState state = taskStateMachine.getState(); List<FailureInfo> failures = ImmutableList.of(); if (state == TaskState.Failed) { failures = toFailures(taskStateMachine.getFailureCauses()); } return new TaskInfo(taskStateMachine.getTaskId(),nextTaskInfoVersion.getAndIncrement(),state,location,DateTime.Now(),sharedBuffer.getInfo(),ImmutableSet.<PlanNodeId> of(),taskContext.getTaskStats(),failures,taskContext.getoutputItems()); }
@JsonCreator public QueryInfo( @JsonProperty("queryId") QueryId queryId,@JsonProperty("session") SessionRepresentation session,@JsonProperty("state") QueryState state,@JsonProperty("memoryPool") MemoryPoolId memoryPool,@JsonProperty("scheduled") boolean scheduled,@JsonProperty("self") URI self,@JsonProperty("fieldNames") List<String> fieldNames,@JsonProperty("query") String query,@JsonProperty("queryStats") QueryStats queryStats,@JsonProperty("setSessionProperties") Map<String,String> setSessionProperties,@JsonProperty("resetSessionProperties") Set<String> resetSessionProperties,@JsonProperty("startedTransactionId") Optional<TransactionId> startedTransactionId,@JsonProperty("clearTransactionId") boolean clearTransactionId,@JsonProperty("updateType") String updateType,@JsonProperty("outputStage") StageInfo outputStage,@JsonProperty("failureInfo") FailureInfo failureInfo,@JsonProperty("errorCode") ErrorCode errorCode,@JsonProperty("inputs") Set<Input> inputs) { requireNonNull(queryId,"queryId is null"); requireNonNull(session,"session is null"); requireNonNull(state,"state is null"); requireNonNull(self,"self is null"); requireNonNull(fieldNames,"fieldNames is null"); requireNonNull(queryStats,"queryStats is null"); requireNonNull(setSessionProperties,"setSessionProperties is null"); requireNonNull(resetSessionProperties,"resetSessionProperties is null"); requireNonNull(startedTransactionId,"startedTransactionId is null"); requireNonNull(query,"query is null"); requireNonNull(inputs,"inputs is null"); this.queryId = queryId; this.session = session; this.state = state; this.memoryPool = requireNonNull(memoryPool,"memoryPool is null"); this.scheduled = scheduled; this.self = self; this.fieldNames = ImmutableList.copyOf(fieldNames); this.query = query; this.queryStats = queryStats; this.setSessionProperties = ImmutableMap.copyOf(setSessionProperties); this.resetSessionProperties = ImmutableSet.copyOf(resetSessionProperties); this.startedTransactionId = startedTransactionId; this.clearTransactionId = clearTransactionId; this.updateType = updateType; this.outputStage = outputStage; this.failureInfo = failureInfo; this.errorType = errorCode == null ? null : toErrorType(errorCode.getCode()); this.errorCode = errorCode; this.inputs = ImmutableSet.copyOf(inputs); }
@Nullable @JsonProperty public FailureInfo getFailureInfo() { return failureInfo; }
public void completionEvent(QueryInfo queryInfo) { try { QueryStats queryStats = queryInfo.getQueryStats(); FailureInfo failureInfo = queryInfo.getFailureInfo(); String failureType = failureInfo == null ? null : failureInfo.getType(); String failureMessage = failureInfo == null ? null : failureInfo.getMessage(); ImmutableMap.Builder<String,String> mergedProperties = ImmutableMap.builder(); mergedProperties.putAll(queryInfo.getSession().getSystemProperties()); for (Map.Entry<String,Map<String,String>> catalogEntry : queryInfo.getSession().getCatalogProperties().entrySet()) { for (Map.Entry<String,String> entry : catalogEntry.getValue().entrySet()) { mergedProperties.put(catalogEntry.getKey() + "." + entry.getKey(),entry.getValue()); } } TaskInfo task = null; StageInfo stageInfo = queryInfo.getoutputStage(); if (stageInfo != null) { task = stageInfo.getTasks().stream() .filter(taskInfo -> taskInfo.getState() == TaskState.Failed) .findFirst().orElse(null); } String failureHost = task == null ? null : task.getSelf().getHost(); String failureTask = task == null ? null : task.getTaskId().toString(); eventClient.post( new QueryCompletionEvent( queryInfo.getQueryId(),queryInfo.getSession().getTransactionId().map(TransactionId::toString).orElse(null),queryInfo.getSession().getUser(),queryInfo.getSession().getPrincipal().orElse(null),queryInfo.getSession().getSource().orElse(null),serverVersion,environment,queryInfo.getSession().getCatalog().orElse(null),queryInfo.getSession().getSchema().orElse(null),queryInfo.getSession().getRemoteUserAddress().orElse(null),queryInfo.getSession().getUserAgent().orElse(null),queryInfo.getState(),queryInfo.getSelf(),queryInfo.getFieldNames(),queryInfo.getQuery(),queryStats.getPeakMemoryReservation().toBytes(),queryStats.getCreateTime(),queryStats.getExecutionStartTime(),queryStats.getEndTime(),queryStats.getQueuedTime(),queryStats.getAnalysisTime(),queryStats.getdistributedplanningTime(),queryStats.getTotalScheduledTime(),queryStats.getTotalcpuTime(),queryStats.getRawInputDataSize(),queryStats.getRawInputPositions(),queryStats.getTotalDrivers(),queryInfo.getErrorCode(),failureType,failureMessage,failureTask,failureHost,objectMapper.writeValueAsstring(queryInfo.getoutputStage()),objectMapper.writeValueAsstring(queryInfo.getFailureInfo()),objectMapper.writeValueAsstring(queryInfo.getInputs()),objectMapper.writeValueAsstring(mergedProperties.build()) ) ); logQueryTimeline(queryInfo); } catch (JsonProcessingException e) { throw Throwables.propagate(e); } }
private static void assertState(QueryStateMachine stateMachine,QueryState expectedState) { assertEquals(stateMachine.getQueryId(),QUERY_ID); assertEqualSessions(stateMachine.getSession().withoutTransactionId(),TEST_SESSION); assertSame(stateMachine.getMemoryPool(),MEMORY_POOL); assertEquals(stateMachine.getSetSessionProperties(),SET_SESSION_PROPERTIES); assertEquals(stateMachine.getResetSessionProperties(),RESET_SESSION_PROPERTIES); QueryInfo queryInfo = stateMachine.getQueryInfo(null); assertEquals(queryInfo.getQueryId(),QUERY_ID); assertEquals(queryInfo.getSelf(),LOCATION); assertNull(queryInfo.getoutputStage()); assertEquals(queryInfo.getQuery(),QUERY); assertEquals(queryInfo.getInputs(),INPUTS); assertEquals(queryInfo.getFieldNames(),OUTPUT_FIELD_NAMES); assertEquals(queryInfo.getUpdateType(),UPDATE_TYPE); assertEquals(queryInfo.getMemoryPool(),MEMORY_POOL.getId()); QueryStats queryStats = queryInfo.getQueryStats(); if (queryInfo.getState() == QUEUED) { assertNull(queryStats.getQueuedTime()); assertNull(queryStats.getTotalPlanningTime()); assertNull(queryStats.getExecutionStartTime()); assertNull(queryStats.getFinishingTime()); assertNull(queryStats.getEndTime()); } else if (queryInfo.getState() == PLANNING) { assertNotNull(queryStats.getQueuedTime()); assertNull(queryStats.getTotalPlanningTime()); assertNull(queryStats.getExecutionStartTime()); assertNull(queryStats.getFinishingTime()); assertNull(queryStats.getEndTime()); } else if (queryInfo.getState() == STARTING) { assertNotNull(queryStats.getQueuedTime()); assertNotNull(queryStats.getTotalPlanningTime()); assertNull(queryStats.getExecutionStartTime()); assertNull(queryStats.getFinishingTime()); assertNull(queryStats.getEndTime()); } else if (queryInfo.getState() == RUNNING) { assertNotNull(queryStats.getQueuedTime()); assertNotNull(queryStats.getTotalPlanningTime()); assertNotNull(queryStats.getExecutionStartTime()); assertNull(queryStats.getFinishingTime()); assertNull(queryStats.getEndTime()); } else if (queryInfo.getState() == FINISHING) { assertNotNull(queryStats.getQueuedTime()); assertNotNull(queryStats.getTotalPlanningTime()); assertNotNull(queryStats.getExecutionStartTime()); assertNull(queryStats.getFinishingTime()); assertNull(queryStats.getEndTime()); } else { assertNotNull(queryStats.getQueuedTime()); assertNotNull(queryStats.getTotalPlanningTime()); assertNotNull(queryStats.getExecutionStartTime()); assertNotNull(queryStats.getFinishingTime()); assertNotNull(queryStats.getEndTime()); } assertEquals(stateMachine.getQueryState(),expectedState); assertEquals(queryInfo.getState(),expectedState); assertEquals(stateMachine.isDone(),expectedState.isDone()); if (expectedState == Failed) { FailureInfo failure = queryInfo.getFailureInfo(); assertNotNull(failure); assertEquals(failure.getMessage(),Failed_CAUSE.getMessage()); assertEquals(failure.getType(),Failed_CAUSE.getClass().getName()); } else { assertNull(queryInfo.getFailureInfo()); } }
protected void updateJobInfo( Set<Table> usedTables,List<Column> columns,QueryStats queryStats,JobState state,QueryError error,List<List<Object>> outputPreview,boolean postUpdate) { if ((usedTables != null) && (usedTables.size() > 0)) { job.getTablesUsed().addAll(usedTables); } if ((columns != null) && (columns.size() > 0)) { job.setColumns(columns); } if (queryStats != null) { job.setQueryStats(queryStats); } if ((state != null) && (job.getState() != JobState.FINISHED) && (job.getState() != JobState.Failed)) { job.setState(state); } if (error != null) { FailureInfo failureInfo = new FailureInfo( error.getFailureInfo().getType(),error.getFailureInfo().getMessage(),Collections.<FailureInfo>emptyList(),Collections.<String>emptyList(),error.getFailureInfo().getErrorLocation()); QueryError queryError = new QueryError( error.getMessage(),error.getsqlState(),error.getErrorCode(),error.getErrorName(),error.getErrorType(),error.getErrorLocation(),failureInfo); job.setError(queryError); } if (postUpdate) { eventBus.post(new JobUpdateEvent(job,outputPreview)); } }
com.facebook.presto.client.ClientSession的实例源码
public ClientSession toClientSession(URI server,boolean debug,Duration clientRequestTimeout) { ImmutableMap.Builder<String,String> properties = ImmutableMap.builder(); properties.putAll(systemProperties); for (Entry<String,Map<String,String>> catalogProperties : this.catalogProperties.entrySet()) { String catalog = catalogProperties.getKey(); for (Entry<String,String> entry : catalogProperties.getValue().entrySet()) { properties.put(catalog + "." + entry.getKey(),entry.getValue()); } } return new ClientSession( requireNonNull(server,"server is null"),identity.getUser(),source.orElse(null),catalog.orElse(null),schema.orElse(null),timeZoneKey.getId(),locale,properties.build(),transactionId.map(TransactionId::toString).orElse(null),debug,clientRequestTimeout); }
@POST @Produces(MediaType.APPLICATION_JSON) public Response createquery(String query,@Context HttpServletRequest servletRequest) { assertRequest(!isNullOrEmpty(query),"sql query is empty"); Session session = createSessionForRequest(servletRequest,accessControl,sessionPropertyManager,queryIdGenerator.createNextQueryId()); ClientSession clientSession = session.toClientSession(serverUri(),false,new Duration(2,MINUTES)); StatementClient client = new StatementClient(httpClient,queryResultsCodec,clientSession,query); List<Column> columns = getColumns(client); Iterator<List<Object>> iterator = flatten(new ResultsPageIterator(client)); SimpleQueryResults results = new SimpleQueryResults(columns,iterator); return Response.ok(results,MediaType.APPLICATION_JSON_TYPE).build(); }
StatementClient startQuery(String sql) { URI uri = createHttpUri(address); String source = firstNonNull(clientInfo.get("ApplicationName"),"presto-jdbc"); ClientSession session = new ClientSession( uri,user,source,catalog.get(),schema.get(),timeZoneId.get(),locale.get(),ImmutableMap.copyOf(sessionProperties),transactionId.get(),MINUTES)); return queryExecutor.startQuery(session,sql); }
public ParallelQueryRunner(int maxParallelism,URI server,String catalog,String schema,int timeout,Duration clientRequestTimeout) { executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("query-runner-%s"))); ImmutableList.Builder<QueryRunner> runners = ImmutableList.builder(); for (int i = 0; i < maxParallelism; i++) { ClientSession session = new ClientSession( server,"test-" + i,"presto-perf",catalog,schema,TimeZone.getDefault().getID(),Locale.getDefault(),ImmutableMap.<String,String>of(),null,clientRequestTimeout); runners.add(new QueryRunner(session,executor,timeout)); } this.runners = runners.build(); }
private static Request buildQueryRequest(ClientSession session,String query) { Request.Builder builder = preparePost() .setUri(uriBuilderFrom(session.getServer()).replacePath("/v1/execute").build()) .setBodyGenerator(createStaticBodyGenerator(query,UTF_8)); if (session.getUser() != null) { builder.setHeader(PrestoHeaders.PRESTO_USER,session.getUser()); } if (session.getSource() != null) { builder.setHeader(PrestoHeaders.PRESTO_SOURCE,session.getSource()); } if (session.getCatalog() != null) { builder.setHeader(PrestoHeaders.PRESTO_CATALOG,session.getCatalog()); } if (session.getSchema() != null) { builder.setHeader(PrestoHeaders.PRESTO_SCHEMA,session.getSchema()); } builder.setHeader(PrestoHeaders.PRESTO_TIME_ZONE,session.getTimeZoneId()); builder.setHeader(USER_AGENT,USER_AGENT_VALUE); return builder.build(); }
public QueryRunner( ClientSession session,JsonCodec<QueryResults> queryResultsCodec,Optional<HostAndPort> socksProxy,Optional<String> keystorePath,Optional<String> keystorePassword,Optional<String> kerberosPrincipal,Optional<String> kerberosRemoteServiceName,boolean authenticationEnabled,KerberosConfig kerberosConfig) { this.session = new atomicreference<>(requireNonNull(session,"session is null")); this.queryResultsCodec = requireNonNull(queryResultsCodec,"queryResultsCodec is null"); this.httpClient = new JettyHttpClient( getHttpClientConfig(socksProxy,keystorePath,keystorePassword,kerberosPrincipal,kerberosRemoteServiceName,authenticationEnabled),kerberosConfig,Optional.<JettyIoPool>empty(),ImmutableList.<HttpRequestFilter>of()); }
public static QueryRunner create( ClientSession session,KerberosConfig kerberosConfig) { return new QueryRunner( session,jsonCodec(QueryResults.class),socksProxy,authenticationEnabled,kerberosConfig); }
private StatementStats execute(ClientSession session,String name,String query) { // start query StatementClient client = new StatementClient(httpClient,session,query); // read query output while (client.isValid() && client.advance()) { // we do not process the output } // verify final state if (client.isClosed()) { throw new IllegalStateException("Query aborted by user"); } if (client.isGone()) { throw new IllegalStateException("Query is gone (server restarted?)"); } QueryError resultsError = client.finalResults().getError(); if (resultsError != null) { RuntimeException cause = null; if (resultsError.getFailureInfo() != null) { cause = resultsError.getFailureInfo().toException(); } throw new BenchmarkDriverExecutionException(format("Query %s Failed: %s",name,resultsError.getMessage()),cause); } return client.finalResults().getStats(); }
public BenchmarkDriver(BenchmarkResultsstore resultsstore,ClientSession clientSession,Iterable<BenchmarkQuery> queries,int warm,int runs,int maxFailures,Optional<HostAndPort> socksProxy) { this.resultsstore = requireNonNull(resultsstore,"resultsstore is null"); this.clientSession = requireNonNull(clientSession,"clientSession is null"); this.queries = ImmutableList.copyOf(requireNonNull(queries,"queries is null")); queryRunner = new BenchmarkQueryRunner(warm,runs,maxFailures,clientSession.getServer(),socksProxy); }
public void run(Suite suite) throws Exception { // select queries to run List<BenchmarkQuery> queries = suite.selectQueries(this.queries); if (queries.isEmpty()) { return; } ClientSession session = ClientSession.withSessionProperties(clientSession,suite.getSessionProperties()); // select schemas to use List<BenchmarkSchema> benchmarkSchemas; if (!suite.getSchemaNameTemplates().isEmpty()) { List<String> schemas = queryRunner.getSchemas(session); benchmarkSchemas = suite.selectSchemas(schemas); } else { benchmarkSchemas = ImmutableList.of(new BenchmarkSchema(session.getSchema())); } if (benchmarkSchemas.isEmpty()) { return; } for (BenchmarkSchema benchmarkSchema : benchmarkSchemas) { for (BenchmarkQuery benchmarkQuery : queries) { session = ClientSession.withCatalogAndSchema(session,session.getCatalog(),benchmarkSchema.getName()); BenchmarkQueryResult result = queryRunner.execute(suite,benchmarkQuery); resultsstore.store(benchmarkSchema,result); } } }
public ClientSession getClientSession() { return new ClientSession( parseServer(server),"presto-benchmark",toProperties(this.sessionProperties),clientRequestTimeout); }
public QueryRunner(ClientSession session,ListeningExecutorService executor,int timeout) { this.session = session; this.executor = executor; HttpClientConfig clientConfig = new HttpClientConfig(); clientConfig.setConnectTimeout(new Duration(10,TimeUnit.SECONDS)); clientConfig.setIdleTimeout(new Duration(timeout,TimeUnit.SECONDS)); httpClient = new JettyHttpClient(clientConfig); }
@Override public void run() { ClientSession session = clientOptions.toClientSession(); KerberosConfig kerberosConfig = clientOptions.toKerberosConfig(); boolean hasQuery = !Strings.isNullOrEmpty(clientOptions.execute); boolean isFromFile = !Strings.isNullOrEmpty(clientOptions.file); if (!hasQuery || !isFromFile) { AnsiConsole.systemInstall(); } initializeLogging(clientOptions.logLevelsFile); String query = clientOptions.execute; if (hasQuery) { query += ";"; } if (isFromFile) { if (hasQuery) { throw new RuntimeException("both --execute and --file specified"); } try { query = Files.toString(new File(clientOptions.file),UTF_8); hasQuery = true; } catch (IOException e) { throw new RuntimeException(format("Error reading from file %s: %s",clientOptions.file,e.getMessage())); } } try (QueryRunner queryRunner = QueryRunner.create( session,Optional.ofNullable(clientOptions.socksProxy),Optional.ofNullable(clientOptions.keystorePath),Optional.ofNullable(clientOptions.keystorePassword),Optional.ofNullable(clientOptions.krb5Principal),Optional.ofNullable(clientOptions.krb5RemoteServiceName),clientOptions.authenticationEnabled,kerberosConfig)) { if (hasQuery) { executeCommand(queryRunner,query,clientOptions.outputFormat); } else { runconsole(queryRunner,session); } } }
static ClientSession processSessionParameterChange(Object parsedStatement,ClientSession session,String> existingProperties) { if (parsedStatement instanceof Use) { Use use = (Use) parsedStatement; session = withCatalogAndSchema(session,use.getCatalog().orElse(session.getCatalog()),use.getSchema()); session = withSessionProperties(session,existingProperties); } return session; }
private static void process(QueryRunner queryRunner,String sql,OutputFormat outputFormat,boolean interactive) { try (Query query = queryRunner.startQuery(sql)) { query.renderOutput(System.out,outputFormat,interactive); ClientSession session = queryRunner.getSession(); // update session properties if present if (!query.getSetSessionProperties().isEmpty() || !query.getResetSessionProperties().isEmpty()) { Map<String,String> sessionProperties = new HashMap<>(session.getProperties()); sessionProperties.putAll(query.getSetSessionProperties()); sessionProperties.keySet().removeAll(query.getResetSessionProperties()); session = withProperties(session,sessionProperties); } // update transaction ID if necessary if (query.isClearTransactionId()) { session = stripTransactionId(session); } if (query.getStartedTransactionId() != null) { session = withTransactionId(session,query.getStartedTransactionId()); } queryRunner.setSession(session); } catch (RuntimeException e) { System.err.println("Error running command: " + e.getMessage()); if (queryRunner.getSession().isDebug()) { e.printstacktrace(); } } }
public ClientSession toClientSession() { return new ClientSession( parseServer(server),toProperties(sessionProperties),clientRequestTimeout); }
@Test public void testDefault() { ClientSession session = new ClientOptions().toClientSession(); assertEquals(session.getServer().toString(),"http://localhost:8080"); assertEquals(session.getSource(),"presto-cli"); }
@Test public void testSource() { ClientOptions options = new ClientOptions(); options.source = "test"; ClientSession session = options.toClientSession(); assertEquals(session.getSource(),"test"); }
@Test public void testServerHostOnly() { ClientOptions options = new ClientOptions(); options.server = "localhost"; ClientSession session = options.toClientSession(); assertEquals(session.getServer().toString(),"http://localhost:80"); }
@Test public void testServerHostPort() { ClientOptions options = new ClientOptions(); options.server = "localhost:8888"; ClientSession session = options.toClientSession(); assertEquals(session.getServer().toString(),"http://localhost:8888"); }
@Test public void testServerHttpUri() { ClientOptions options = new ClientOptions(); options.server = "http://localhost/foo"; ClientSession session = options.toClientSession(); assertEquals(session.getServer().toString(),"http://localhost/foo"); }
@Test public void testServerHttpsUri() { ClientOptions options = new ClientOptions(); options.server = "https://localhost/foo"; ClientSession session = options.toClientSession(); assertEquals(session.getServer().toString(),"https://localhost/foo"); }
@Test public void testUpdateSessionParameters() throws Exception { ClientOptions options = new ClientOptions(); ClientSession session = options.toClientSession(); sqlParser sqlParser = new sqlParser(); ImmutableMap<String,String> existingProperties = ImmutableMap.of("query_max_memory","10GB","distributed_join","true"); session = Console.processSessionParameterChange(sqlParser.createStatement("USE test_catalog.test_schema"),existingProperties); assertEquals(session.getCatalog(),"test_catalog"); assertEquals(session.getSchema(),"test_schema"); assertEquals(session.getProperties().get("query_max_memory"),"10GB"); assertEquals(session.getProperties().get("distributed_join"),"true"); session = Console.processSessionParameterChange(sqlParser.createStatement("USE test_schema_b"),"test_schema_b"); assertEquals(session.getProperties().get("query_max_memory"),"true"); session = Console.processSessionParameterChange(sqlParser.createStatement("USE test_catalog_2.test_schema"),"test_catalog_2"); assertEquals(session.getSchema(),"true"); }
@Test public void testAutoCompleteWithoutSchema() { ClientSession session = new ClientOptions().toClientSession(); QueryRunner runner = QueryRunner.create(session,Optional.<HostAndPort>empty(),Optional.<String>empty(),null); TableNameCompleter completer = new TableNameCompleter(runner); assertEquals(completer.complete("SELECT is_infi",14,ImmutableList.of()),7); }
@Inject public PrestoRakamRaptormetastore( @Named("presto.metastore.jdbc") JDBCPoolDataSource prestometastoreDataSource,EventBus eventBus,ProjectConfig projectConfig,PrestoConfig prestoConfig) { super(eventBus); dbi = new DBI(prestometastoreDataSource); dbi.registerMapper(new TableColumn.Mapper(new SignatureReferenceTypeManager())); this.dao = ondemandDao(dbi,MetadataDao.class); this.projectConfig = projectConfig; this.prestoConfig = prestoConfig; defaultSession = new ClientSession( prestoConfig.getAddress(),"rakam","api-server",ImmutableSet.of(),prestoConfig.getColdStorageConnector(),"default",TimeZone.getTimeZone(ZoneOffset.UTC).getID(),ENGLISH,ImmutableMap.of(),Duration.succinctDuration(1,MINUTES)); activeNodeCount = suppliers.memoizeWithExpiration(() -> { String getNodeCount = "select count(*) from system.runtime.nodes where state = 'active'"; QueryResult queryResult = new PrestoQueryExecution(defaultSession,getNodeCount,false).getResult().join(); if (queryResult.isFailed()) { throw new RakamException(queryResult.getError().message,SERVICE_UNAVAILABLE); } return Ints.checkedCast((long) queryResult.getResult().get(0).get(0)); },5,MINUTES); }
public ClientSession createSession(String catalog,ZoneId timezone,String> sessionProperties,String user) { return new ClientSession( prestoConfig.getAddress(),user == null ? "rakam" : user,catalog == null ? "default" : catalog,TimeZone.getTimeZone(timezone == null ? ZoneOffset.UTC : timezone).getID(),Locale.ENGLISH,sessionProperties,new Duration(1,TimeUnit.MINUTES)); }
public ClientSession create(String user,String schema) { return new ClientSession(server.get(),timeZoneId,clientSessionTimeout ); }
public ClientSession create(String schema) { return new ClientSession(server.get(),clientSessionTimeout ); }
public ClientSession create() { return new ClientSession(server.get(),defaultSchema,clientSessionTimeout ); }
public List<String> getSchemas(ClientSession session) { failures = 0; while (true) { // start query StatementClient client = new StatementClient(httpClient,"show schemas"); // read query output ImmutableList.Builder<String> schemas = ImmutableList.builder(); while (client.isValid() && client.advance()) { // we do not process the output Iterable<List<Object>> data = client.current().getData(); if (data != null) { for (List<Object> objects : data) { schemas.add(objects.get(0).toString()); } } } // verify final state if (client.isClosed()) { throw new IllegalStateException("Query aborted by user"); } if (client.isGone()) { throw new IllegalStateException("Query is gone (server restarted?)"); } QueryError resultsError = client.finalResults().getError(); if (resultsError != null) { RuntimeException cause = null; if (resultsError.getFailureInfo() != null) { cause = resultsError.getFailureInfo().toException(); } handleFailure(cause); continue; } return schemas.build(); } }
public StatementClient startQuery(ClientSession session,String query) { return new StatementClient(httpClient,queryInfoCodec,query); }
public ClientSession getSession() { return session.get(); }
public void setSession(ClientSession session) { this.session.set(requireNonNull(session,"session is null")); }
public PrestoQueryExecution internalExecuterawQuery(RequestContext context,String query,boolean update) { return new PrestoQueryExecution(clientSession,update); }
protected QueryRunner(ClientSession session,HttpClient httpClient) { this.session = checkNotNull(session,"session is null"); this.queryResultsCodec = checkNotNull(queryResultsCodec,"queryResultsCodec is null"); this.httpClient = httpClient; }
com.facebook.presto.client.Column的实例源码
@Override public void addResults(QueryResults results) { if (types.get() == null && results.getColumns() != null) { types.set(getTypes(results.getColumns())); } if (results.getData() != null) { checkState(types.get() != null,"Data without types received!"); List<Column> columns = results.getColumns(); for (List<Object> fields : results.getData()) { ImmutableMap.Builder<String,Object> builder = ImmutableMap.builder(); for (int i = 0; i < fields.size(); i++) { Type type = types.get().get(i); Object value = convertValue(fields.get(i),type); if (value != null) { builder.put(columns.get(i).getName(),value); } } producer.send(new KeyedMessage<>(topicName,count.getAndIncrement(),builder.build())); } } }
@POST @Produces(MediaType.APPLICATION_JSON) public Response createquery(String query,@Context HttpServletRequest servletRequest) { assertRequest(!isNullOrEmpty(query),"sql query is empty"); Session session = createSessionForRequest(servletRequest,accessControl,sessionPropertyManager,queryIdGenerator.createNextQueryId()); ClientSession clientSession = session.toClientSession(serverUri(),false,new Duration(2,MINUTES)); StatementClient client = new StatementClient(httpClient,queryResultsCodec,clientSession,query); List<Column> columns = getColumns(client); Iterator<List<Object>> iterator = flatten(new ResultsPageIterator(client)); SimpleQueryResults results = new SimpleQueryResults(columns,iterator); return Response.ok(results,MediaType.APPLICATION_JSON_TYPE).build(); }
private static List<Column> createColumnsList(QueryInfo queryInfo) { requireNonNull(queryInfo,"queryInfo is null"); StageInfo outputStage = queryInfo.getoutputStage(); requireNonNull(outputStage,"outputStage is null"); List<String> names = queryInfo.getFieldNames(); List<Type> types = outputStage.getTypes(); checkArgument(names.size() == types.size(),"names and types size mismatch"); ImmutableList.Builder<Column> list = ImmutableList.builder(); for (int i = 0; i < names.size(); i++) { String name = names.get(i); TypeSignature typeSignature = types.get(i).getTypeSignature(); String type = typeSignature.toString(); list.add(new Column(name,type,new ClientTypeSignature(typeSignature))); } return list.build(); }
private static List<Column> getColumns(StatementClient client) throws sqlException { while (client.isValid()) { List<Column> columns = client.current().getColumns(); if (columns != null) { return columns; } client.advance(); } QueryResults results = client.finalResults(); if (!client.isFailed()) { throw new sqlException(format("Query has no columns (#%s)",results.getId())); } throw resultsException(results); }
@JsonCreator public Job(@JsonProperty("user") final String user,@JsonProperty("query") final String query,@JsonProperty("uuid") final UUID uuid,@JsonProperty("output") final PersistentJobOutput output,@JsonProperty("queryStats") final QueryStats queryStats,@JsonProperty("state") final JobState state,@JsonProperty("columns") final List<Column> columns,@JsonProperty("tablesUsed") final Set<Table> tablesUsed,@JsonProperty("queryStarted") final DateTime queryStarted,@JsonProperty("error") final QueryError error,@JsonProperty("queryFinished") final DateTime queryFinished) { this.user = user; this.query = query; this.uuid = uuid; this.output = output; this.queryStats = queryStats; this.state = state; this.columns = columns; this.tablesUsed = tablesUsed; this.queryStarted = queryStarted; this.error = error; this.queryFinished = queryFinished; }
public Job(final String user,final String query,final UUID uuid,final PersistentJobOutput output,final QueryStats stats,final JobState state,final List<Column> columns,final QueryError error,final DateTime queryFinished) { this(user,query,uuid,output,stats,state,columns,Sets.<Table>newConcurrentHashSet(),new DateTime(),error,queryFinished ); }
@Override public TableRow map(int index,ResultSet r,StatementContext ctx) throws sqlException { try { return new TableRow( r.getLong("id"),r.getString("connector_id"),r.getString("schema_"),r.getString("table_"),objectMapper.<List<Column>>readValue(r.getString("columns"),columnTypeReference)); } catch (IOException e) { log.error("Caught exception mapping TableRow",e); return null; } }
protected Function<Column,Type> columnTypeGetter() { return column -> { String typeName = column.getType(); Type type = prestoServer.getMetadata().getType(parseTypeSignature(typeName)); if (type == null) { throw new AssertionError("Unhandled type: " + typeName); } return type; }; }
private static List<Column> getColumns(StatementClient client) { while (client.isValid()) { List<Column> columns = client.current().getColumns(); if (columns != null) { return columns; } client.advance(); } if (!client.isFailed()) { throw internalServerError("No columns"); } throw internalServerError(failureMessage(client.finalResults())); }
PrestoResultSet(StatementClient client) throws sqlException { this.client = requireNonNull(client,"client is null"); this.sessionTimeZone = DateTimeZone.forID(client.getTimeZoneId()); this.queryId = client.current().getId(); List<Column> columns = getColumns(client); this.fieldMap = getFieldMap(columns); this.columnInfoList = getColumnInfo(columns); this.resultSetMetaData = new PrestoResultSetMetaData(columnInfoList); this.results = flatten(new ResultsPageIterator(client)); }
private static Map<String,Integer> getFieldMap(List<Column> columns) { Map<String,Integer> map = new HashMap<>(); for (int i = 0; i < columns.size(); i++) { String name = columns.get(i).getName().toLowerCase(ENGLISH); if (!map.containsKey(name)) { map.put(name,i + 1); } } return ImmutableMap.copyOf(map); }
private void doRenderResults(PrintStream out,OutputFormat format,boolean interactive,List<Column> columns) throws IOException { List<String> fieldNames = Lists.transform(columns,Column::getName); if (interactive) { pageOutput(format,fieldNames); } else { sendOutput(out,format,fieldNames); } }
@Override public void addColumns(List<Column> columns) throws FiletooLargeException { if (!headerWritten && includeHeader) { List<String> columnNames = Lists.transform(columns,Column::getName); writeCsvRow(columnNames.toArray(new String[columnNames.size()])); headerWritten = true; } }
private List<HiveColumn> queryColumns(String query) { final ImmutableList.Builder<HiveColumn> cache = ImmutableList.builder(); QueryRunner queryRunner = queryRunnerFactory.create(); QueryClient queryClient = new QueryClient(queryRunner,io.dropwizard.util.Duration.seconds(60),query); try { queryClient.executeWith(new Function<StatementClient,Void>() { @Nullable @Override public Void apply(StatementClient client) { QueryResults results = client.current(); if (results.getData() != null) { for (List<Object> row : results.getData()) { Column column = new Column((String) row.get(0),(String) row.get(1),new ClientTypeSignature(TypeSignature.parseTypeSignature((String)row.get(1)))); boolean isNullable = (Boolean) row.get(2); boolean isPartition = (Boolean) row.get(3); cache.add(HiveColumn.fromColumn(column,isNullable,isPartition)); } } return null; } }); } catch (QueryClient.QueryTimeOutException e) { log.error("Caught timeout loading columns",e); } return cache.build(); }
/** * Rate Limited updateJobInfo */ protected void rlUpdateJobInfo( Set<Table> usedTables,List<Column> columns,QueryStats queryStats,JobState state,QueryError error,List<List<Object>> outputPreview) { if (updateLimiter.tryAcquire(1)) { updateJobInfo(usedTables,queryStats,outputPreview,true); } else { updateJobInfo(usedTables,false); } }
private static List<Column> createColumnsList(QueryInfo queryInfo) { checkNotNull(queryInfo,"queryInfo is null"); StageInfo outputStage = queryInfo.getoutputStage(); if (outputStage == null) { checkNotNull(outputStage,"outputStage is null"); } List<String> names = queryInfo.getFieldNames(); ArrayList<Type> types = new ArrayList<>(); for (TupleInfo tupleInfo : outputStage.getTupleInfos()) { types.addAll(tupleInfo.getTypes()); } checkArgument(names.size() == types.size(),"names and types size mismatch"); ImmutableList.Builder<Column> list = ImmutableList.builder(); for (int i = 0; i < names.size(); i++) { String name = names.get(i); Type type = types.get(i); switch (type) { case BOOLEAN: list.add(new Column(name,"boolean")); break; case FIXED_INT_64: list.add(new Column(name,"bigint")); break; case DOUBLE: list.add(new Column(name,"double")); break; case VARIABLE_BINARY: list.add(new Column(name,"varchar")); break; default: throw new IllegalArgumentException("unhandled type: " + type); } } return list.build(); }
@Override public void addResults(QueryResults results) { if (types.get() == null && results.getColumns() != null) { types.set(getTypes(results.getColumns())); } if (results.getData() != null) { checkState(types.get() != null,"Data without types received!"); List<Column> columns = results.getColumns(); for (List<Object> fields : results.getData()) { String redisKey = tableName + ":" + count.getAndIncrement(); try (Jedis jedis = jedisPool.getResource()) { switch (dataFormat) { case "string": ImmutableMap.Builder<String,Object> builder = ImmutableMap.builder(); for (int i = 0; i < fields.size(); i++) { Type type = types.get().get(i); Object value = convertValue(fields.get(i),type); if (value != null) { builder.put(columns.get(i).getName(),value); } } jedis.set(redisKey,jsonEncoder.toString(builder.build())); break; case "hash": // add keys to zset String redisZset = "keyset:" + tableName; jedis.zadd(redisZset,count.get(),redisKey); // add values to Hash for (int i = 0; i < fields.size(); i++) { jedis.hset(redisKey,columns.get(i).getName(),fields.get(i).toString()); } break; default: throw new AssertionError("unhandled value type: " + dataFormat); } } } } }
protected List<Type> getTypes(List<Column> columns) { return ImmutableList.copyOf(transform(columns,columnTypeGetter())); }
public SimpleQueryResults(List<Column> columns,Iterator<List<Object>> data) { this.columns = requireNonNull(columns,"columns is null"); this.data = requireNonNull(data,"data is null"); }
@JsonProperty public List<Column> getColumns() { return columns; }
@GET @Path("queries") public Response getUserQueries( @Auth AirpalUser user,@PathParam("id") String userId,@QueryParam("results") int numResults,@QueryParam("table") List<PartitionedTable> tables) { Iterable<Job> recentlyRun; int results = Optional.of(numResults).or(0); if (results <= 0) { results = 100; } if (tables.size() < 1) { recentlyRun = jobHistoryStore.getRecentlyRunForUser(userId,results); } else { recentlyRun = jobHistoryStore.getRecentlyRunForUser( userId,results,Iterables.transform(tables,new PartitionedTabletoTable())); } ImmutableList.Builder<Job> filtered = ImmutableList.builder(); for (Job job : recentlyRun) { if (job.getTablesUsed().isEmpty() && (job.getState() == JobState.Failed)) { filtered.add(job); continue; } for (Table table : job.getTablesUsed()) { if (AuthorizationUtil.isAuthorizedRead(user,table)) { filtered.add(new Job( job.getUser(),job.getQuery(),job.getUuid(),job.getoutput(),job.getQueryStats(),job.getState(),Collections.<Column>emptyList(),Collections.<Table>emptySet(),job.getQueryStartedDateTime(),job.getError(),job.getQueryFinishedDateTime())); } } } List<Job> sortedResult = Ordering .natural() .nullsLast() .onResultOf(JOB_ORDERING) .reverse() .immutableSortedcopy(filtered.build()); return Response.ok(sortedResult).build(); }
@GET public Response getQueries( @Auth AirpalUser user,@QueryParam("table") List<PartitionedTable> tables) { Iterable<Job> recentlyRun; int results = Optional.of(numResults).or(200); if (tables.size() < 1) { recentlyRun = jobHistoryStore.getRecentlyRun(results); } else { recentlyRun = jobHistoryStore.getRecentlyRun( results,new PartitionedTable.PartitionedTabletoTable())); } ImmutableList.Builder<Job> filtered = ImmutableList.builder(); for (Job job : recentlyRun) { if (job.getTablesUsed().isEmpty() && (job.getState() == JobState.Failed)) { filtered.add(job); continue; } for (Table table : job.getTablesUsed()) { if (AuthorizationUtil.isAuthorizedRead(user,job.getQueryFinishedDateTime())); } } } List<Job> sortedResult = Ordering .natural() .nullsLast() .onResultOf(JOB_ORDERING) .reverse() .immutableSortedcopy(filtered.build()); return Response.ok(sortedResult).build(); }
@GET @Path("history") @Produces(MediaType.APPLICATION_JSON) public Response getHistory( @Auth AirpalUser user,@QueryParam("table") List<Table> tables) { Iterable<Job> recentlyRun; if (tables.size() < 1) { recentlyRun = jobHistoryStore.getRecentlyRun(200); } else { Table[] tablesArray = tables.toArray(new Table[tables.size()]); Table[] restTables = Arrays.copyOfRange(tablesArray,1,tablesArray.length); recentlyRun = jobHistoryStore.getRecentlyRun(200,tablesArray[0],restTables); } ImmutableList.Builder<Job> filtered = ImmutableList.builder(); for (Job job : recentlyRun) { if (job.getTablesUsed().isEmpty() && (job.getState() == JobState.Failed)) { filtered.add(job); continue; } for (Table table : job.getTablesUsed()) { if (AuthorizationUtil.isAuthorizedRead(user,job.getQueryFinishedDateTime())); } } } List<Job> sortedResult = Ordering .natural() .nullsLast() .onResultOf(JOB_ORDERING) .reverse() .immutableSortedcopy(filtered.build()); return Response.ok(sortedResult).build(); }
public void addColumns(List<Column> columns) throws FiletooLargeException;
@Override public void addColumns(List<Column> columns) throws FiletooLargeException {}
public static HivePartition fromColumn(Column column,List<Object> values) { return new HivePartition(column.getName(),column.getType(),values); }
public static HiveColumn fromColumn(Column column,boolean isNullable,boolean isPartition) { return new HiveColumn(column.getName(),isPartition,isNullable); }
public TableRowMapper(ObjectMapper objectMapper) { this.objectMapper = objectMapper; this.columnTypeReference = new TypeReference<List<Column>>() {}; }
protected void updateJobInfo( Set<Table> usedTables,List<List<Object>> outputPreview,boolean postUpdate) { if ((usedTables != null) && (usedTables.size() > 0)) { job.getTablesUsed().addAll(usedTables); } if ((columns != null) && (columns.size() > 0)) { job.setColumns(columns); } if (queryStats != null) { job.setQueryStats(queryStats); } if ((state != null) && (job.getState() != JobState.FINISHED) && (job.getState() != JobState.Failed)) { job.setState(state); } if (error != null) { FailureInfo failureInfo = new FailureInfo( error.getFailureInfo().getType(),error.getFailureInfo().getMessage(),null,Collections.<FailureInfo>emptyList(),Collections.<String>emptyList(),error.getFailureInfo().getErrorLocation()); QueryError queryError = new QueryError( error.getMessage(),error.getsqlState(),error.getErrorCode(),error.getErrorName(),error.getErrorType(),error.getErrorLocation(),failureInfo); job.setError(queryError); } if (postUpdate) { eventBus.post(new JobUpdateEvent(job,outputPreview)); } }
com.facebook.presto.client.QueryError的实例源码
@JsonCreator public Job(@JsonProperty("user") final String user,@JsonProperty("query") final String query,@JsonProperty("uuid") final UUID uuid,@JsonProperty("output") final PersistentJobOutput output,@JsonProperty("queryStats") final QueryStats queryStats,@JsonProperty("state") final JobState state,@JsonProperty("columns") final List<Column> columns,@JsonProperty("tablesUsed") final Set<Table> tablesUsed,@JsonProperty("queryStarted") final DateTime queryStarted,@JsonProperty("error") final QueryError error,@JsonProperty("queryFinished") final DateTime queryFinished) { this.user = user; this.query = query; this.uuid = uuid; this.output = output; this.queryStats = queryStats; this.state = state; this.columns = columns; this.tablesUsed = tablesUsed; this.queryStarted = queryStarted; this.error = error; this.queryFinished = queryFinished; }
public Job(final String user,final String query,final UUID uuid,final PersistentJobOutput output,final QueryStats stats,final JobState state,final List<Column> columns,final QueryError error,final DateTime queryFinished) { this(user,query,uuid,output,stats,state,columns,Sets.<Table>newConcurrentHashSet(),new DateTime(),error,queryFinished ); }
private static QueryError toQueryError(QueryInfo queryInfo) { FailureInfo failure = queryInfo.getFailureInfo(); if (failure == null) { QueryState state = queryInfo.getState(); if ((!state.isDone()) || (state == QueryState.FINISHED)) { return null; } log.warn("Query %s in state %s has no failure info",queryInfo.getQueryId(),state); failure = toFailure(new RuntimeException(format("Query is %s (reason unkNown)",state))).toFailureInfo(); } ErrorCode errorCode; if (queryInfo.getErrorCode() != null) { errorCode = queryInfo.getErrorCode(); } else { errorCode = INTERNAL_ERROR.toErrorCode(); log.warn("Failed query %s has no error code",queryInfo.getQueryId()); } return new QueryError( failure.getMessage(),null,errorCode.getCode(),errorCode.getName(),toErrorType(errorCode.getCode()).toString(),failure.getErrorLocation(),failure); }
private StatementStats execute(ClientSession session,String name,String query) { // start query StatementClient client = new StatementClient(httpClient,queryResultsCodec,session,query); // read query output while (client.isValid() && client.advance()) { // we do not process the output } // verify final state if (client.isClosed()) { throw new IllegalStateException("Query aborted by user"); } if (client.isGone()) { throw new IllegalStateException("Query is gone (server restarted?)"); } QueryError resultsError = client.finalResults().getError(); if (resultsError != null) { RuntimeException cause = null; if (resultsError.getFailureInfo() != null) { cause = resultsError.getFailureInfo().toException(); } throw new BenchmarkDriverExecutionException(format("Query %s Failed: %s",name,resultsError.getMessage()),cause); } return client.finalResults().getStats(); }
static sqlException resultsException(QueryResults results) { QueryError error = requireNonNull(results.getError()); String message = format("Query Failed (#%s): %s",results.getId(),error.getMessage()); Throwable cause = (error.getFailureInfo() == null) ? null : error.getFailureInfo().toException(); return new sqlException(message,error.getsqlState(),error.getErrorCode(),cause); }
public void renderFailure(PrintStream out) { QueryResults results = client.finalResults(); QueryError error = results.getError(); checkState(error != null); out.printf("Query %s Failed: %s%n",error.getMessage()); if (client.isDebug() && (error.getFailureInfo() != null)) { error.getFailureInfo().toException().printstacktrace(out); } if (error.getErrorLocation() != null) { renderErrorLocation(client.getQuery(),error.getErrorLocation(),out); } out.println(); }
/** * Rate Limited updateJobInfo */ protected void rlUpdateJobInfo( Set<Table> usedTables,List<Column> columns,QueryStats queryStats,JobState state,QueryError error,List<List<Object>> outputPreview) { if (updateLimiter.tryAcquire(1)) { updateJobInfo(usedTables,queryStats,outputPreview,true); } else { updateJobInfo(usedTables,false); } }
public List<String> getSchemas(ClientSession session) { failures = 0; while (true) { // start query StatementClient client = new StatementClient(httpClient,"show schemas"); // read query output ImmutableList.Builder<String> schemas = ImmutableList.builder(); while (client.isValid() && client.advance()) { // we do not process the output Iterable<List<Object>> data = client.current().getData(); if (data != null) { for (List<Object> objects : data) { schemas.add(objects.get(0).toString()); } } } // verify final state if (client.isClosed()) { throw new IllegalStateException("Query aborted by user"); } if (client.isGone()) { throw new IllegalStateException("Query is gone (server restarted?)"); } QueryError resultsError = client.finalResults().getError(); if (resultsError != null) { RuntimeException cause = null; if (resultsError.getFailureInfo() != null) { cause = resultsError.getFailureInfo().toException(); } handleFailure(cause); continue; } return schemas.build(); } }
protected void updateJobInfo( Set<Table> usedTables,List<List<Object>> outputPreview,boolean postUpdate) { if ((usedTables != null) && (usedTables.size() > 0)) { job.getTablesUsed().addAll(usedTables); } if ((columns != null) && (columns.size() > 0)) { job.setColumns(columns); } if (queryStats != null) { job.setQueryStats(queryStats); } if ((state != null) && (job.getState() != JobState.FINISHED) && (job.getState() != JobState.Failed)) { job.setState(state); } if (error != null) { FailureInfo failureInfo = new FailureInfo( error.getFailureInfo().getType(),error.getFailureInfo().getMessage(),Collections.<FailureInfo>emptyList(),Collections.<String>emptyList(),error.getFailureInfo().getErrorLocation()); QueryError queryError = new QueryError( error.getMessage(),error.getErrorName(),error.getErrorType(),failureInfo); job.setError(queryError); } if (postUpdate) { eventBus.post(new JobUpdateEvent(job,outputPreview)); } }
com.facebook.presto.client.QueryResults的实例源码
@Override public void addResults(QueryResults results) { if (types.get() == null && results.getColumns() != null) { types.set(getTypes(results.getColumns())); } if (results.getData() != null) { checkState(types.get() != null,"Data without types received!"); List<Column> columns = results.getColumns(); for (List<Object> fields : results.getData()) { ImmutableMap.Builder<String,Object> builder = ImmutableMap.builder(); for (int i = 0; i < fields.size(); i++) { Type type = types.get().get(i); Object value = convertValue(fields.get(i),type); if (value != null) { builder.put(columns.get(i).getName(),value); } } producer.send(new KeyedMessage<>(topicName,count.getAndIncrement(),builder.build())); } } }
@Override public void addResults(QueryResults results) { if (!loggedUri.getAndSet(true)) { log.info("Query %s: %s?pretty",results.getId(),results.getInfoUri()); } if (types.get() == null && results.getColumns() != null) { types.set(getTypes(results.getColumns())); } if (results.getData() != null) { checkState(types.get() != null,"data received without types"); rows.addAll(transform(results.getData(),dataToRow(timeZoneKey,types.get()))); } }
@Inject public ExecuteResource( HttpServerInfo serverInfo,AccessControl accessControl,SessionPropertyManager sessionPropertyManager,@ForExecute HttpClient httpClient,QueryIdGenerator queryIdGenerator,JsonCodec<QueryResults> queryResultsCodec) { this.serverInfo = requireNonNull(serverInfo,"serverInfo is null"); this.accessControl = requireNonNull(accessControl,"accessControl is null"); this.sessionPropertyManager = requireNonNull(sessionPropertyManager,"sessionPropertyManager is null"); this.httpClient = requireNonNull(httpClient,"httpClient is null"); this.queryIdGenerator = requireNonNull(queryIdGenerator,"queryIdGenerator is null"); this.queryResultsCodec = requireNonNull(queryResultsCodec,"queryResultsCodec is null"); }
public synchronized QueryResults getResults(long token,UriInfo uriInfo,Duration maxWaitTime) throws InterruptedException { // is the a repeated request for the last results? String requestedpath = uriInfo.getAbsolutePath().getPath(); if (lastResultPath != null && requestedpath.equals(lastResultPath)) { // tell query manager we are still interested in the query queryManager.getQueryInfo(queryId); queryManager.recordHeartbeat(queryId); return lastResult; } if (token < resultId.get()) { throw new WebApplicationException(Status.GONE); } // if this is not a request for the next results,return not found if (lastResult.getNextUri() == null || !requestedpath.equals(lastResult.getNextUri().getPath())) { // unkNown token throw new WebApplicationException(Status.NOT_FOUND); } return getNextResults(uriInfo,maxWaitTime); }
public BenchmarkQueryRunner(int warm,int runs,boolean debug,int maxFailures,URI serverUri,Optional<HostAndPort> socksProxy) { checkArgument(warm >= 0,"warm is negative"); this.warm = warm; checkArgument(runs >= 1,"runs must be at least 1"); this.runs = runs; checkArgument(maxFailures >= 0,"maxFailures must be at least 0"); this.maxFailures = maxFailures; this.debug = debug; this.queryResultsCodec = jsonCodec(QueryResults.class); requireNonNull(socksProxy,"socksProxy is null"); HttpClientConfig httpClientConfig = new HttpClientConfig(); if (socksProxy.isPresent()) { httpClientConfig.setSocksProxy(socksProxy.get()); } this.httpClient = new JettyHttpClient(httpClientConfig.setConnectTimeout(new Duration(10,TimeUnit.SECONDS))); nodes = getAllNodes(requireNonNull(serverUri,"serverUri is null")); }
private static List<Column> getColumns(StatementClient client) throws sqlException { while (client.isValid()) { List<Column> columns = client.current().getColumns(); if (columns != null) { return columns; } client.advance(); } QueryResults results = client.finalResults(); if (!client.isFailed()) { throw new sqlException(format("Query has no columns (#%s)",results.getId())); } throw resultsException(results); }
public QueryRunner( ClientSession session,JsonCodec<QueryResults> queryResultsCodec,Optional<HostAndPort> socksProxy,Optional<String> keystorePath,Optional<String> keystorePassword,Optional<String> kerberosPrincipal,Optional<String> kerberosRemoteServiceName,boolean authenticationEnabled,KerberosConfig kerberosConfig) { this.session = new atomicreference<>(requireNonNull(session,"session is null")); this.queryResultsCodec = requireNonNull(queryResultsCodec,"queryResultsCodec is null"); this.httpClient = new JettyHttpClient( getHttpClientConfig(socksProxy,keystorePath,keystorePassword,kerberosPrincipal,kerberosRemoteServiceName,authenticationEnabled),kerberosConfig,Optional.<JettyIoPool>empty(),ImmutableList.<HttpRequestFilter>of()); }
public static QueryRunner create( ClientSession session,KerberosConfig kerberosConfig) { return new QueryRunner( session,jsonCodec(QueryResults.class),socksProxy,authenticationEnabled,kerberosConfig); }
private List<String> queryMetadata(String query) { ImmutableList.Builder<String> cache = ImmutableList.builder(); try (StatementClient client = queryRunner.startInternalQuery(query)) { while (client.isValid() && !Thread.currentThread().isInterrupted()) { QueryResults results = client.current(); if (results.getData() != null) { for (List<Object> row : results.getData()) { cache.add((String) row.get(0)); } } client.advance(); } } return cache.build(); }
private static Response getQueryResults(Query query,Optional<Long> token,Duration wait) throws InterruptedException { QueryResults queryResults; if (token.isPresent()) { queryResults = query.getResults(token.get(),uriInfo,wait); } else { queryResults = query.getNextResults(uriInfo,wait); } ResponseBuilder response = Response.ok(queryResults); // add set session properties query.getSetSessionProperties().entrySet().stream() .forEach(entry -> response.header(PRESTO_SET_SESSION,entry.getKey() + '=' + entry.getValue())); // add clear session properties query.getResetSessionProperties().stream() .forEach(name -> response.header(PRESTO_CLEAR_SESSION,name)); // add new transaction ID query.getStartedTransactionId() .ifPresent(transactionId -> response.header(PRESTO_STARTED_TRANSACTION_ID,transactionId)); // add clear transaction ID directive if (query.isClearTransactionId()) { response.header(PRESTO_CLEAR_TRANSACTION_ID,true); } return response.build(); }
private QueryExecutor(String userAgent,JsonCodec<ServerInfo> serverInfoCodec,HostAndPort socksProxy) { requireNonNull(userAgent,"userAgent is null"); requireNonNull(queryResultsCodec,"queryResultsCodec is null"); requireNonNull(serverInfoCodec,"serverInfoCodec is null"); this.queryInfoCodec = queryResultsCodec; this.serverInfoCodec = serverInfoCodec; this.httpClient = new JettyHttpClient( new HttpClientConfig() .setConnectTimeout(new Duration(10,TimeUnit.SECONDS)) .setSocksProxy(socksProxy),new JettyIoPool("presto-jdbc",new JettyIoPoolConfig()),ImmutableSet.of(new UserAgentRequestFilter(userAgent))); }
static sqlException resultsException(QueryResults results) { QueryError error = requireNonNull(results.getError()); String message = format("Query Failed (#%s): %s",error.getMessage()); Throwable cause = (error.getFailureInfo() == null) ? null : error.getFailureInfo().toException(); return new sqlException(message,error.getsqlState(),error.getErrorCode(),cause); }
private void renderUpdate(PrintStream out,QueryResults results) { String status = results.getUpdateType(); if (results.getUpdateCount() != null) { long count = results.getUpdateCount(); status += format(": %s row%s",count,(count != 1) ? "s" : ""); } out.println(status); discardResults(); }
public void renderFailure(PrintStream out) { QueryResults results = client.finalResults(); QueryError error = results.getError(); checkState(error != null); out.printf("Query %s Failed: %s%n",error.getMessage()); if (client.isDebug() && (error.getFailureInfo() != null)) { error.getFailureInfo().toException().printstacktrace(out); } if (error.getErrorLocation() != null) { renderErrorLocation(client.getQuery(),error.getErrorLocation(),out); } out.println(); }
private List<HiveColumn> queryColumns(String query) { final ImmutableList.Builder<HiveColumn> cache = ImmutableList.builder(); QueryRunner queryRunner = queryRunnerFactory.create(); QueryClient queryClient = new QueryClient(queryRunner,io.dropwizard.util.Duration.seconds(60),query); try { queryClient.executeWith(new Function<StatementClient,Void>() { @Nullable @Override public Void apply(StatementClient client) { QueryResults results = client.current(); if (results.getData() != null) { for (List<Object> row : results.getData()) { Column column = new Column((String) row.get(0),(String) row.get(1),new ClientTypeSignature(TypeSignature.parseTypeSignature((String)row.get(1)))); boolean isNullable = (Boolean) row.get(2); boolean isPartition = (Boolean) row.get(3); cache.add(HiveColumn.fromColumn(column,isNullable,isPartition)); } } return null; } }); } catch (QueryClient.QueryTimeOutException e) { log.error("Caught timeout loading columns",e); } return cache.build(); }
private List<List<Object>> queryRows(String query) { final ImmutableList.Builder<List<Object>> cache = ImmutableList.builder(); QueryRunner queryRunner = queryRunnerFactory.create(); QueryClient queryClient = new QueryClient(queryRunner,Void>() { @Nullable @Override public Void apply(StatementClient client) { QueryResults results = client.current(); if (results.getData() != null) { cache.addAll(results.getData()); } return null; } }); } catch (QueryClient.QueryTimeOutException e) { log.error("Caught timeout loading columns",e); } return cache.build(); }
@Override public void addResults(QueryResults results) { if (types.get() == null && results.getColumns() != null) { types.set(getTypes(results.getColumns())); } if (results.getData() != null) { checkState(types.get() != null,"Data without types received!"); List<Column> columns = results.getColumns(); for (List<Object> fields : results.getData()) { String redisKey = tableName + ":" + count.getAndIncrement(); try (Jedis jedis = jedisPool.getResource()) { switch (dataFormat) { case "string": ImmutableMap.Builder<String,Object> builder = ImmutableMap.builder(); for (int i = 0; i < fields.size(); i++) { Type type = types.get().get(i); Object value = convertValue(fields.get(i),type); if (value != null) { builder.put(columns.get(i).getName(),value); } } jedis.set(redisKey,jsonEncoder.toString(builder.build())); break; case "hash": // add keys to zset String redisZset = "keyset:" + tableName; jedis.zadd(redisZset,count.get(),redisKey); // add values to Hash for (int i = 0; i < fields.size(); i++) { jedis.hset(redisKey,columns.get(i).getName(),fields.get(i).toString()); } break; default: throw new AssertionError("unhandled value type: " + dataFormat); } } } } }
private static String failureMessage(QueryResults results) { return format("Query Failed (#%s): %s",results.getError().getMessage()); }
@Test public void testQuery() throws Exception { // start query Request request = preparePost() .setUri(uriFor("/v1/statement")) .setBodyGenerator(createStaticBodyGenerator("show catalogs",UTF_8)) .setHeader(PRESTO_USER,"user") .setHeader(PRESTO_SOURCE,"source") .setHeader(PRESTO_CATALOG,"catalog") .setHeader(PRESTO_SCHEMA,"schema") .addHeader(PRESTO_SESSION,QUERY_MAX_MEMORY + "=1GB") .addHeader(PRESTO_SESSION,distributeD_JOIN + "=true," + HASH_PARTITION_COUNT + " = 43") .build(); QueryResults queryResults = client.execute(request,createJsonResponseHandler(jsonCodec(QueryResults.class))); // get the query info QueryInfo queryInfo = server.getQueryManager().getQueryInfo(new QueryId(queryResults.getId())); // verify session properties assertEquals(queryInfo.getSession().getSystemProperties(),ImmutableMap.builder() .put(QUERY_MAX_MEMORY,"1GB") .put(distributeD_JOIN,"true") .put(HASH_PARTITION_COUNT,"43") .build()); ImmutableList.Builder<List<Object>> data = ImmutableList.builder(); if (queryResults.getData() != null) { data.addAll(queryResults.getData()); } while (queryResults.getNextUri() != null) { queryResults = client.execute(prepareGet().setUri(queryResults.getNextUri()).build(),createJsonResponseHandler(jsonCodec(QueryResults.class))); if (queryResults.getData() != null) { data.addAll(queryResults.getData()); } } // only the system catalog exists by default List<List<Object>> rows = data.build(); assertEquals(rows,ImmutableList.of(ImmutableList.of("system"))); }
static QueryExecutor create(String userAgent) { return new QueryExecutor(userAgent,jsonCodec(ServerInfo.class),getSystemSocksProxy()); }
private void renderQueryOutput(PrintStream out,OutputFormat outputFormat,boolean interactive) { StatusPrinter statusPrinter = null; @SuppressWarnings("resource") PrintStream errorChannel = interactive ? out : System.err; if (interactive) { statusPrinter = new StatusPrinter(client,out); statusPrinter.printinitialStatusUpdates(); } else { waitForData(); } if ((!client.isFailed()) && (!client.isGone()) && (!client.isClosed())) { QueryResults results = client.isValid() ? client.current() : client.finalResults(); if (results.getUpdateType() != null) { renderUpdate(out,results); } else if (results.getColumns() == null) { errorChannel.printf("Query %s has no columns\n",results.getId()); return; } else { renderResults(out,outputFormat,interactive,results.getColumns()); } } if (statusPrinter != null) { statusPrinter.printFinalInfo(); } if (client.isClosed()) { errorChannel.println("Query aborted by user"); } else if (client.isGone()) { errorChannel.println("Query is gone (server restarted?)"); } else if (client.isFailed()) { renderFailure(errorChannel); } }
public void printFinalInfo() { Duration wallTime = nanosSince(start); QueryResults results = client.finalResults(); StatementStats stats = results.getStats(); int nodes = stats.getNodes(); if ((nodes == 0) || (stats.getTotalSplits() == 0)) { return; } // blank line out.println(); // Query 12,FINISHED,1 node String querySummary = String.format("Query %s,%s,%,d %s",stats.getState(),nodes,pluralize("node",nodes)); out.println(querySummary); if (debug) { out.println(results.getInfoUri() + "?pretty"); } // Splits: 1000 total,842 done (84.20%) String splitsSummary = String.format("Splits: %,d total,d done (%.2f%%)",stats.getTotalSplits(),stats.getCompletedSplits(),percentage(stats.getCompletedSplits(),stats.getTotalSplits())); out.println(splitsSummary); if (debug) { // cpu Time: 565.2s total,26K rows/s,3.85MB/s Duration cpuTime = millis(stats.getcpuTimeMillis()); String cpuTimeSummary = String.format("cpu Time: %.1fs total,%5s rows/s,%8s,%d%% active",cpuTime.getValue(SECONDS),formatCountRate(stats.getProcessedRows(),cpuTime,false),formatDatarate(bytes(stats.getProcessedBytes()),true),(int) percentage(stats.getcpuTimeMillis(),stats.getWallTimeMillis())); out.println(cpuTimeSummary); double parallelism = cpuTime.getValue(MILLISECONDS) / wallTime.getValue(MILLISECONDS); // Per Node: 3.5 parallelism,83.3K rows/s,0.7 MB/s String perNodeSummary = String.format("Per Node: %.1f parallelism,%8s",parallelism / nodes,formatCountRate((double) stats.getProcessedRows() / nodes,wallTime,formatDatarate(bytes(stats.getProcessedBytes() / nodes),true)); reprintLine(perNodeSummary); out.println(String.format("Parallelism: %.1f",parallelism)); } // 0:32 [2.12GB,15M rows] [67MB/s,463K rows/s] String statsLine = String.format("%s [%s rows,%s] [%s rows/s,%s]",formatTime(wallTime),formatCount(stats.getProcessedRows()),formatDataSize(bytes(stats.getProcessedBytes()),true)); out.println(statsLine); // blank line out.println(); }
protected QueryRunner(ClientSession session,HttpClient httpClient) { this.session = checkNotNull(session,"session is null"); this.queryResultsCodec = checkNotNull(queryResultsCodec,"queryResultsCodec is null"); this.httpClient = httpClient; }
public QueryRunner create(String user,String schema) { return new QueryRunner(sessionFactory.create(user,schema),httpClient); }
public QueryRunner create() { return new QueryRunner(sessionFactory.create(),httpClient); }
private Map<String,List<String>> queryMetadata(String query) { final Map<String,List<String>> cache = Maps.newHashMap(); QueryRunner queryRunner = queryRunnerFactory.create(); QueryClient queryClient = new QueryClient(queryRunner,Void>() { @Nullable @Override public Void apply(StatementClient client) { QueryResults results = client.current(); if (results.getData() != null) { for (List<Object> row : results.getData()) { String schema = (String) row.get(1); String table = (String) row.get(2); if (EXCLUDED_SCHEMAS.contains(schema)) { continue; } List<String> tables = cache.get(schema); if (tables == null) { tables = Lists.newArrayList(); cache.put(schema,tables); } tables.add(table); } } return null; } }); } catch (QueryClient.QueryTimeOutException e) { log.error("Caught timeout loading columns",e); } return ImmutableMap.copyOf(cache); }
public QueryResults finalResults() { return finalResults.get(); }
void addResults(QueryResults result);
今天关于com.facebook.presto.client.FailureInfo的实例源码和facebook源代码的讲解已经结束,谢谢您的阅读,如果想了解更多关于com.facebook.presto.client.ClientSession的实例源码、com.facebook.presto.client.Column的实例源码、com.facebook.presto.client.QueryError的实例源码、com.facebook.presto.client.QueryResults的实例源码的相关知识,请在本站搜索。
本文标签: